TAJO-2110: Fix incorrect DateTime and remove hard coded tests.
Closes #993
diff --git a/CHANGES b/CHANGES
index b729cce..8351ee4 100644
--- a/CHANGES
+++ b/CHANGES
@@ -137,6 +137,8 @@
BUG FIXES
+ TAJO-2110: Fix incorrect DateTime and remove hard coded tests. (jinho)
+
TAJO-2119: Invalid sort result when sort key columns contain non-ascii values.
(jihoon)
diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java
index 76990f2..0ab0c6f 100644
--- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java
+++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java
@@ -28,13 +28,10 @@
import org.apache.tajo.catalog.partition.PartitionDesc;
import org.apache.tajo.catalog.partition.PartitionMethodDesc;
import org.apache.tajo.catalog.proto.CatalogProtos;
-import org.apache.tajo.catalog.proto.CatalogProtos.PartitionKeyProto;
-import org.apache.tajo.catalog.proto.CatalogProtos.SchemaProto;
-import org.apache.tajo.catalog.proto.CatalogProtos.DataFormat;
-import org.apache.tajo.catalog.proto.CatalogProtos.TableDescProto;
-import org.apache.tajo.catalog.proto.CatalogProtos.TableIdentifierProto;
+import org.apache.tajo.catalog.proto.CatalogProtos.*;
import org.apache.tajo.common.TajoDataTypes;
import org.apache.tajo.common.TajoDataTypes.DataType;
+import org.apache.tajo.conf.TajoConf;
import org.apache.tajo.exception.InvalidOperationException;
import org.apache.tajo.exception.UndefinedOperatorException;
import org.apache.tajo.storage.StorageConstants;
@@ -321,8 +318,8 @@
}
}
- public static TableMeta newTableMeta(String dataFormat) {
- KeyValueSet defaultProperties = CatalogUtil.newDefaultProperty(dataFormat);
+ public static TableMeta newTableMeta(String dataFormat, TajoConf conf) {
+ KeyValueSet defaultProperties = CatalogUtil.newDefaultProperty(dataFormat, conf);
return new TableMeta(dataFormat, defaultProperties);
}
@@ -890,13 +887,32 @@
return pair;
}
- /*
+ /**
* It is the relationship graph of type conversions.
* It contains tuples, each of which (LHS type, RHS type, Result type).
*/
public static final Map<Type, Map<Type, Type>> OPERATION_CASTING_MAP = Maps.newHashMap();
+ /**
+ * It is the casting direction of relationship graph
+ */
+ private static final Map<Type, Map<Type, Direction>> CASTING_DIRECTION_MAP = Maps.newHashMap();
+
+ public static Direction getCastingDirection(Type lhs, Type rhs) {
+ Direction direction = TUtil.getFromNestedMap(CatalogUtil.CASTING_DIRECTION_MAP, lhs, rhs);
+ if (direction == null) {
+ return Direction.BOTH;
+ }
+ return direction;
+ }
+
+ public enum Direction {
+ LHS,
+ RHS,
+ BOTH
+ }
+
static {
// Type Conversion Map
TUtil.putToNestedMap(OPERATION_CASTING_MAP, Type.BOOLEAN, Type.BOOLEAN, Type.BOOLEAN);
@@ -962,7 +978,18 @@
TUtil.putToNestedMap(OPERATION_CASTING_MAP, Type.TIMESTAMP, Type.VARCHAR, Type.TEXT);
TUtil.putToNestedMap(OPERATION_CASTING_MAP, Type.TIME, Type.TIME, Type.TIME);
+ TUtil.putToNestedMap(OPERATION_CASTING_MAP, Type.TIME, Type.DATE, Type.TIMESTAMP);
+ TUtil.putToNestedMap(CASTING_DIRECTION_MAP, Type.TIME, Type.DATE, Direction.RHS);
+
TUtil.putToNestedMap(OPERATION_CASTING_MAP, Type.DATE, Type.DATE, Type.DATE);
+ TUtil.putToNestedMap(OPERATION_CASTING_MAP, Type.DATE, Type.TIME, Type.TIMESTAMP);
+ TUtil.putToNestedMap(CASTING_DIRECTION_MAP, Type.DATE, Type.TIME, Direction.LHS);
+
+ TUtil.putToNestedMap(OPERATION_CASTING_MAP, Type.DATE, Type.INTERVAL, Type.TIMESTAMP);
+ TUtil.putToNestedMap(CASTING_DIRECTION_MAP, Type.DATE, Type.INTERVAL, Direction.LHS);
+
+ TUtil.putToNestedMap(OPERATION_CASTING_MAP, Type.INTERVAL, Type.DATE, Type.TIMESTAMP);
+ TUtil.putToNestedMap(CASTING_DIRECTION_MAP, Type.INTERVAL, Type.DATE, Direction.RHS);
TUtil.putToNestedMap(OPERATION_CASTING_MAP, Type.INET4, Type.INET4, Type.INET4);
}
@@ -980,8 +1007,11 @@
* @param dataFormat DataFormat
* @return Table properties
*/
- public static KeyValueSet newDefaultProperty(String dataFormat) {
+ public static KeyValueSet newDefaultProperty(String dataFormat, TajoConf conf) {
KeyValueSet options = new KeyValueSet();
+ // set default timezone to the system timezone
+ options.set(StorageConstants.TIMEZONE, conf.getSystemTimezone().getID());
+
if (dataFormat.equalsIgnoreCase(BuiltinStorages.TEXT)) {
options.set(StorageConstants.TEXT_DELIMITER, StorageConstants.DEFAULT_FIELD_DELIMITER);
} else if (dataFormat.equalsIgnoreCase("JSON")) {
diff --git a/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestTableDesc.java b/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestTableDesc.java
index 085975b..f599406 100644
--- a/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestTableDesc.java
+++ b/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestTableDesc.java
@@ -19,11 +19,13 @@
package org.apache.tajo.catalog;
import org.apache.hadoop.fs.Path;
+import org.apache.tajo.BuiltinStorages;
import org.apache.tajo.catalog.json.CatalogGsonHelper;
import org.apache.tajo.catalog.proto.CatalogProtos;
import org.apache.tajo.catalog.statistics.ColumnStats;
import org.apache.tajo.catalog.statistics.TableStats;
import org.apache.tajo.common.TajoDataTypes.Type;
+import org.apache.tajo.conf.TajoConf;
import org.apache.tajo.util.CommonTestingUtil;
import org.junit.Before;
import org.junit.Test;
@@ -33,6 +35,7 @@
import static org.junit.Assert.assertEquals;
public class TestTableDesc {
+ TajoConf conf = new TajoConf();
TableMeta info;
Schema schema;
TableDesc desc;
@@ -45,7 +48,7 @@
.add("name", Type.BLOB)
.add("addr", Type.TEXT)
.build();
- info = CatalogUtil.newTableMeta("TEXT");
+ info = CatalogUtil.newTableMeta(BuiltinStorages.TEXT, conf);
path = new Path(CommonTestingUtil.getTestDir(), "table1");
desc = new TableDesc("table1", schema, info, path.toUri());
stats = new TableStats();
@@ -72,7 +75,7 @@
.add("name", Type.BLOB)
.add("addr", Type.TEXT)
.build();
- TableMeta info = CatalogUtil.newTableMeta("TEXT");
+ TableMeta info = CatalogUtil.newTableMeta(BuiltinStorages.TEXT, conf);
testClone(info);
Path path = new Path(CommonTestingUtil.getTestDir(), "tajo");
diff --git a/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestTableMeta.java b/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestTableMeta.java
index c6f4d1a..124b5ee 100644
--- a/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestTableMeta.java
+++ b/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestTableMeta.java
@@ -21,6 +21,7 @@
import org.apache.tajo.BuiltinStorages;
import org.apache.tajo.catalog.json.CatalogGsonHelper;
import org.apache.tajo.catalog.proto.CatalogProtos.TableProto;
+import org.apache.tajo.conf.TajoConf;
import org.apache.tajo.rpc.protocolrecords.PrimitiveProtos;
import org.junit.Before;
import org.junit.Test;
@@ -28,16 +29,17 @@
import static org.junit.Assert.*;
public class TestTableMeta {
+ TajoConf conf = new TajoConf();
TableMeta meta = null;
-
+
@Before
public void setUp() {
- meta = CatalogUtil.newTableMeta("TEXT");
+ meta = CatalogUtil.newTableMeta(BuiltinStorages.TEXT, conf);
}
@Test
public void testTableMetaTableProto() {
- TableMeta meta1 = CatalogUtil.newTableMeta("TEXT");
+ TableMeta meta1 = CatalogUtil.newTableMeta(BuiltinStorages.TEXT, conf);
TableMeta meta2 = new TableMeta(meta1.getProto());
assertEquals(meta1, meta2);
@@ -45,7 +47,7 @@
@Test
public final void testClone() throws CloneNotSupportedException {
- TableMeta meta1 = CatalogUtil.newTableMeta("TEXT");
+ TableMeta meta1 = CatalogUtil.newTableMeta(BuiltinStorages.TEXT, conf);
TableMeta meta2 = (TableMeta) meta1.clone();
assertEquals(meta1.getDataFormat(), meta2.getDataFormat());
@@ -54,7 +56,7 @@
@Test
public void testSchema() throws CloneNotSupportedException {
- TableMeta meta1 = CatalogUtil.newTableMeta("TEXT");
+ TableMeta meta1 = CatalogUtil.newTableMeta(BuiltinStorages.TEXT, conf);
TableMeta meta2 = (TableMeta) meta1.clone();
assertEquals(meta1, meta2);
}
@@ -66,7 +68,7 @@
@Test
public void testEqualsObject() {
- TableMeta meta2 = CatalogUtil.newTableMeta("TEXT");
+ TableMeta meta2 = CatalogUtil.newTableMeta(BuiltinStorages.TEXT, conf);
assertTrue(meta.equals(meta2));
assertNotSame(meta, meta2);
}
@@ -79,7 +81,7 @@
int MAX_COUNT = 17;
- TableMeta meta1 = CatalogUtil.newTableMeta(BuiltinStorages.TEXT);
+ TableMeta meta1 = CatalogUtil.newTableMeta(BuiltinStorages.TEXT, conf);
for (int i = 0; i < MAX_COUNT; i++) {
meta1.putProperty("key"+i, "value"+i);
}
diff --git a/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java b/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java
index 9c1e430..14d6e30 100644
--- a/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java
+++ b/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java
@@ -20,6 +20,7 @@
import com.google.common.collect.Sets;
import org.apache.hadoop.fs.Path;
+import org.apache.tajo.BuiltinStorages;
import org.apache.tajo.TajoConstants;
import org.apache.tajo.catalog.dictionary.InfoSchemaMetadataDictionary;
import org.apache.tajo.catalog.partition.PartitionDesc;
@@ -433,7 +434,7 @@
String tableName = "indexed";
- TableMeta meta = CatalogUtil.newTableMeta("TEXT");
+ TableMeta meta = CatalogUtil.newTableMeta(BuiltinStorages.TEXT, server.getConf());
return new TableDesc(
CatalogUtil.buildFQName(TajoConstants.DEFAULT_DATABASE_NAME, tableName), relationSchema, meta,
new Path(CommonTestingUtil.getTestDir(), "indexed").toUri());
diff --git a/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalogAgainstCaseSensitivity.java b/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalogAgainstCaseSensitivity.java
index 6af2798..6b7e458 100644
--- a/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalogAgainstCaseSensitivity.java
+++ b/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalogAgainstCaseSensitivity.java
@@ -18,6 +18,7 @@
package org.apache.tajo.catalog;
+import org.apache.tajo.BuiltinStorages;
import org.apache.tajo.catalog.partition.PartitionDesc;
import org.apache.tajo.catalog.proto.CatalogProtos.*;
import org.apache.tajo.catalog.proto.CatalogProtos.AlterTablespaceProto.AlterTablespaceCommand;
@@ -352,7 +353,7 @@
TableDesc tableDesc = new TableDesc(
CatalogUtil.buildFQName(databaseName, tableName),
tableSchema,
- CatalogUtil.newTableMeta("JSON"),
+ CatalogUtil.newTableMeta(BuiltinStorages.JSON, server.getConf()),
URI.create("hdfs://xxx.com/json_Table")
);
diff --git a/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/CliClientParamsFactory.java b/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/CliClientParamsFactory.java
index 1d40db3..a4b0c1d 100644
--- a/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/CliClientParamsFactory.java
+++ b/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/CliClientParamsFactory.java
@@ -24,6 +24,7 @@
import java.util.HashMap;
import java.util.Map;
import java.util.Properties;
+import java.util.TimeZone;
class CliClientParamsFactory {
static Map<String, String> DEFAULT_PARAMS = new HashMap<>();
@@ -35,6 +36,7 @@
DEFAULT_PARAMS.put(ClientParameters.RETRY, "3");
DEFAULT_PARAMS.put(ClientParameters.ROW_FETCH_SIZE, "200");
DEFAULT_PARAMS.put(ClientParameters.USE_COMPRESSION, "false");
+ DEFAULT_PARAMS.put(ClientParameters.TIMEZONE, TimeZone.getDefault().getID());
}
public static Properties get(@Nullable Properties connParam) {
diff --git a/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/commands/DescTableCommand.java b/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/commands/DescTableCommand.java
index 6f01911..43fcf2e 100644
--- a/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/commands/DescTableCommand.java
+++ b/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/commands/DescTableCommand.java
@@ -118,7 +118,7 @@
FileUtil.humanReadableByteCount(desc.getStats().getNumBytes(),
true)).append("\n");
}
- sb.append("Options: \n");
+ sb.append("Options:\n");
for(Map.Entry<String, String> entry : desc.getMeta().toMap().entrySet()){
/*
diff --git a/tajo-client/src/main/java/org/apache/tajo/client/CatalogAdminClientImpl.java b/tajo-client/src/main/java/org/apache/tajo/client/CatalogAdminClientImpl.java
index 97f28a6..f1a4712 100644
--- a/tajo-client/src/main/java/org/apache/tajo/client/CatalogAdminClientImpl.java
+++ b/tajo-client/src/main/java/org/apache/tajo/client/CatalogAdminClientImpl.java
@@ -20,7 +20,6 @@
import com.google.protobuf.ServiceException;
import org.apache.tajo.annotation.Nullable;
-import org.apache.tajo.catalog.CatalogUtil;
import org.apache.tajo.catalog.Schema;
import org.apache.tajo.catalog.TableDesc;
import org.apache.tajo.catalog.TableMeta;
@@ -175,7 +174,7 @@
throwsIfThisError(res.getState(), UnavailableTableLocationException.class);
ensureOk(res.getState());
- return CatalogUtil.newTableDesc(res.getTable());
+ return new TableDesc(res.getTable());
}
@Override
@@ -241,7 +240,7 @@
}
ensureOk(res.getState());
- return CatalogUtil.newTableDesc(res.getTable());
+ return new TableDesc(res.getTable());
}
@Override
diff --git a/tajo-client/src/main/java/org/apache/tajo/client/ClientParameterHelper.java b/tajo-client/src/main/java/org/apache/tajo/client/ClientParameterHelper.java
index 8549178..fc5bc7d 100644
--- a/tajo-client/src/main/java/org/apache/tajo/client/ClientParameterHelper.java
+++ b/tajo-client/src/main/java/org/apache/tajo/client/ClientParameterHelper.java
@@ -30,6 +30,7 @@
import static org.apache.tajo.SessionVars.COMPRESSED_RESULT_TRANSFER;
import static org.apache.tajo.SessionVars.FETCH_ROWNUM;
+import static org.apache.tajo.SessionVars.TIMEZONE;
import static org.apache.tajo.client.ClientParameterHelper.ActionType.CONNECTION_PARAM;
import static org.apache.tajo.client.ClientParameterHelper.ActionType.SESSION_UPDATE;
import static org.apache.tajo.rpc.RpcConstants.CLIENT_CONNECTION_TIMEOUT;
@@ -75,6 +76,7 @@
}
});
PARAMETERS.put(ClientParameters.RETRY, new SimpleConnectionParamAction(RpcConstants.CLIENT_RETRY_NUM));
+ PARAMETERS.put(ClientParameters.TIMEZONE, new SimpleSessionAction(TIMEZONE));
}
enum ActionType {
diff --git a/tajo-client/src/main/java/org/apache/tajo/client/ClientParameters.java b/tajo-client/src/main/java/org/apache/tajo/client/ClientParameters.java
index e3ee019..d5069d5 100644
--- a/tajo-client/src/main/java/org/apache/tajo/client/ClientParameters.java
+++ b/tajo-client/src/main/java/org/apache/tajo/client/ClientParameters.java
@@ -29,4 +29,5 @@
String CONNECT_TIMEOUT = "connectTimeout";
String SOCKET_TIMEOUT = "socketTimeout";
String RETRY = "retry";
+ String TIMEZONE = "timezone";
}
diff --git a/tajo-client/src/main/java/org/apache/tajo/client/QueryClientImpl.java b/tajo-client/src/main/java/org/apache/tajo/client/QueryClientImpl.java
index 3618f1f..921da09 100644
--- a/tajo-client/src/main/java/org/apache/tajo/client/QueryClientImpl.java
+++ b/tajo-client/src/main/java/org/apache/tajo/client/QueryClientImpl.java
@@ -26,14 +26,12 @@
import org.apache.tajo.QueryIdFactory;
import org.apache.tajo.SessionVars;
import org.apache.tajo.TajoIdProtos.SessionIdProto;
+import org.apache.tajo.TajoProtos.CodecType;
import org.apache.tajo.TajoProtos.QueryState;
import org.apache.tajo.auth.UserRoleInfo;
-import org.apache.tajo.catalog.CatalogUtil;
-import org.apache.tajo.catalog.Schema;
import org.apache.tajo.catalog.SchemaFactory;
import org.apache.tajo.catalog.TableDesc;
import org.apache.tajo.client.v2.exception.ClientUnableToConnectException;
-import org.apache.tajo.TajoProtos.CodecType;
import org.apache.tajo.exception.*;
import org.apache.tajo.ipc.ClientProtos.*;
import org.apache.tajo.ipc.QueryMasterClientProtocol;
@@ -299,7 +297,7 @@
GetQueryResultResponse response = getResultResponse(queryId);
- TableDesc tableDesc = CatalogUtil.newTableDesc(response.getTableDesc());
+ TableDesc tableDesc = new TableDesc(response.getTableDesc());
return new FetchResultSet(this, tableDesc.getLogicalSchema(), queryId, defaultFetchRows);
}
diff --git a/tajo-client/src/main/java/org/apache/tajo/client/TajoClientUtil.java b/tajo-client/src/main/java/org/apache/tajo/client/TajoClientUtil.java
index 2fe67ed..1d0ce10 100644
--- a/tajo-client/src/main/java/org/apache/tajo/client/TajoClientUtil.java
+++ b/tajo-client/src/main/java/org/apache/tajo/client/TajoClientUtil.java
@@ -77,7 +77,7 @@
public static ResultSet createResultSet(TajoClient client, QueryId queryId,
ClientProtos.GetQueryResultResponse response, int fetchRows)
throws IOException {
- TableDesc desc = CatalogUtil.newTableDesc(response.getTableDesc());
+ TableDesc desc = new TableDesc(response.getTableDesc());
return new FetchResultSet(client, desc.getLogicalSchema(), queryId, fetchRows);
}
diff --git a/tajo-client/src/main/java/org/apache/tajo/jdbc/TajoResultSetBase.java b/tajo-client/src/main/java/org/apache/tajo/jdbc/TajoResultSetBase.java
index 5cc4309..9198a1e 100644
--- a/tajo-client/src/main/java/org/apache/tajo/jdbc/TajoResultSetBase.java
+++ b/tajo-client/src/main/java/org/apache/tajo/jdbc/TajoResultSetBase.java
@@ -22,7 +22,7 @@
import org.apache.tajo.SessionVars;
import org.apache.tajo.catalog.Schema;
import org.apache.tajo.common.TajoDataTypes;
-import org.apache.tajo.datum.*;
+import org.apache.tajo.datum.TimestampDatum;
import org.apache.tajo.storage.Tuple;
import org.apache.tajo.util.datetime.DateTimeUtil;
import org.apache.tajo.util.datetime.TimeMeta;
@@ -223,10 +223,10 @@
case FLOAT8: return tuple.getFloat8(index);
case NUMERIC: return tuple.getFloat8(index);
case DATE: {
- return toDate(tuple.getTimeDate(index), timezone);
+ return toDate(tuple.getTimeDate(index), null);
}
case TIME: {
- return toTime(tuple.getTimeDate(index), timezone);
+ return toTime(tuple.getTimeDate(index), null);
}
case TIMESTAMP: {
return toTimestamp(tuple.getTimeDate(index), timezone);
@@ -268,8 +268,6 @@
switch(tuple.type(index)) {
case BOOLEAN:
return String.valueOf(tuple.getBool(index));
- case TIME:
- return TimeDatum.asChars(tuple.getTimeDate(index), timezone, false);
case TIMESTAMP:
return TimestampDatum.asChars(tuple.getTimeDate(index), timezone, false);
default :
@@ -302,10 +300,7 @@
}
private Date toDate(TimeMeta tm, TimeZone tz) {
- if (tz != null) {
- DateTimeUtil.toUserTimezone(tm, tz);
- }
- return new Date(DateTimeUtil.julianTimeToJavaTime(DateTimeUtil.toJulianTimestamp(tm)));
+ return DateTimeUtil.toJavaDate(tm, tz);
}
@Override
@@ -333,10 +328,7 @@
}
private Time toTime(TimeMeta tm, TimeZone tz) {
- if (tz != null) {
- DateTimeUtil.toUserTimezone(tm, tz);
- }
- return new Time(DateTimeUtil.toJavaTime(tm.hours, tm.minutes, tm.secs, tm.fsecs));
+ return DateTimeUtil.toJavaTime(tm, tz);
}
@Override
@@ -364,10 +356,7 @@
}
private Timestamp toTimestamp(TimeMeta tm, TimeZone tz) {
- if (tz != null) {
- DateTimeUtil.toUserTimezone(tm, tz);
- }
- return new Timestamp(DateTimeUtil.julianTimeToJavaTime(DateTimeUtil.toJulianTimestamp(tm)));
+ return DateTimeUtil.toJavaTimestamp(tm, tz);
}
@Override
diff --git a/tajo-client/src/main/java/org/apache/tajo/jdbc/WaitingResultSet.java b/tajo-client/src/main/java/org/apache/tajo/jdbc/WaitingResultSet.java
index c9967b4..eb2d9cb 100644
--- a/tajo-client/src/main/java/org/apache/tajo/jdbc/WaitingResultSet.java
+++ b/tajo-client/src/main/java/org/apache/tajo/jdbc/WaitingResultSet.java
@@ -20,7 +20,6 @@
import org.apache.tajo.QueryId;
import org.apache.tajo.TajoProtos;
-import org.apache.tajo.catalog.CatalogUtil;
import org.apache.tajo.catalog.Schema;
import org.apache.tajo.catalog.TableDesc;
import org.apache.tajo.client.QueryClient;
@@ -64,7 +63,7 @@
}
ClientProtos.GetQueryResultResponse response = tajoClient.getResultResponse(queryId);
- TableDesc tableDesc = CatalogUtil.newTableDesc(response.getTableDesc());
+ TableDesc tableDesc = new TableDesc(response.getTableDesc());
return tableDesc.getLogicalSchema();
} catch (TajoException e) {
throw SQLExceptionUtil.toSQLException(e);
diff --git a/tajo-cluster-tests/src/test/java/org/apache/tajo/BackendTestingUtil.java b/tajo-cluster-tests/src/test/java/org/apache/tajo/BackendTestingUtil.java
index 28605d3..23aebca 100644
--- a/tajo-cluster-tests/src/test/java/org/apache/tajo/BackendTestingUtil.java
+++ b/tajo-cluster-tests/src/test/java/org/apache/tajo/BackendTestingUtil.java
@@ -40,7 +40,7 @@
.add("deptname", Type.TEXT)
.add("score", Type.INT4)
.build();
- mockupMeta = CatalogUtil.newTableMeta("TEXT");
+ mockupMeta = CatalogUtil.newTableMeta(BuiltinStorages.TEXT, new TajoConf());
}
public static void writeTmpTable(TajoConf conf, Path tablePath)
diff --git a/tajo-cluster-tests/src/test/java/org/apache/tajo/QueryTestCaseBase.java b/tajo-cluster-tests/src/test/java/org/apache/tajo/QueryTestCaseBase.java
index 116990f..aebd7b9 100644
--- a/tajo-cluster-tests/src/test/java/org/apache/tajo/QueryTestCaseBase.java
+++ b/tajo-cluster-tests/src/test/java/org/apache/tajo/QueryTestCaseBase.java
@@ -56,10 +56,7 @@
import org.apache.tajo.storage.BufferPool;
import org.apache.tajo.storage.StorageUtil;
import org.apache.tajo.util.FileUtil;
-import org.junit.AfterClass;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Rule;
+import org.junit.*;
import org.junit.rules.TestName;
import org.junit.rules.TestRule;
import org.junit.rules.TestWatcher;
@@ -265,6 +262,11 @@
name.getMethodName()));
}
+ @After
+ public void clear() {
+ getClient().unsetSessionVariables(Lists.newArrayList(SessionVars.TIMEZONE.name()));
+ }
+
public QueryTestCaseBase() {
// hive 0.12 does not support quoted identifier.
// So, we use lower case database names when Tajo uses HiveCatalogStore.
diff --git a/tajo-cluster-tests/src/test/java/org/apache/tajo/TajoTestingCluster.java b/tajo-cluster-tests/src/test/java/org/apache/tajo/TajoTestingCluster.java
index b1a3306..5413905 100644
--- a/tajo-cluster-tests/src/test/java/org/apache/tajo/TajoTestingCluster.java
+++ b/tajo-cluster-tests/src/test/java/org/apache/tajo/TajoTestingCluster.java
@@ -118,11 +118,6 @@
void initPropertiesAndConfigs() {
- // Set time zone
- TimeZone testDefaultTZ = TimeZone.getTimeZone(TajoConstants.DEFAULT_SYSTEM_TIMEZONE);
- conf.setSystemTimezone(testDefaultTZ);
- TimeZone.setDefault(testDefaultTZ);
-
// Injection of equality testing code of logical plan (de)serialization
conf.setClassVar(ConfVars.LOGICAL_PLAN_REWRITE_RULE_PROVIDER_CLASS, LogicalPlanTestRuleProvider.class);
conf.setClassVar(ConfVars.GLOBAL_PLAN_REWRITE_RULE_PROVIDER_CLASS, GlobalPlanTestRuleProvider.class);
@@ -546,12 +541,6 @@
LOG.info("Minicluster is stopping");
LOG.info("========================================");
- try {
- Thread.sleep(3000);
- } catch (InterruptedException e) {
- e.printStackTrace();
- }
-
shutdownMiniTajoCluster();
if(this.catalogServer != null) {
@@ -560,7 +549,7 @@
}
try {
- Thread.sleep(3000);
+ Thread.sleep(2000);
} catch (InterruptedException e) {
e.printStackTrace();
}
@@ -613,7 +602,7 @@
for (int i = 0; i < names.length; i++) {
createTable(names[i], schemas[i], tableOption, tables[i]);
}
- Thread.sleep(1000);
+
ResultSet res = client.executeQueryAndGetResult(query);
return res;
}
diff --git a/tajo-common/src/main/java/org/apache/tajo/TajoConstants.java b/tajo-common/src/main/java/org/apache/tajo/TajoConstants.java
index f431fad..1b60826 100644
--- a/tajo-common/src/main/java/org/apache/tajo/TajoConstants.java
+++ b/tajo-common/src/main/java/org/apache/tajo/TajoConstants.java
@@ -31,9 +31,6 @@
public static final String DEFAULT_DATABASE_NAME = "default";
public static final String DEFAULT_SCHEMA_NAME = "";
- /** Java 6 only recognize GMT instead of UTC. So, we should keep using GMT. */
- public static final String DEFAULT_SYSTEM_TIMEZONE = "GMT";
-
public static final String EMPTY_STRING = "";
public static final String SYSTEM_HA_DIR_NAME = "ha";
diff --git a/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java b/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java
index 440af80..e619b24 100644
--- a/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java
+++ b/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java
@@ -46,7 +46,6 @@
import java.util.concurrent.TimeUnit;
public class TajoConf extends Configuration {
- private static TimeZone SYSTEM_TIMEZONE;
private static int DATE_ORDER = -1;
private static final Map<String, ConfVars> vars = new HashMap<>();
@@ -79,7 +78,6 @@
addResource(path);
}
- @SuppressWarnings("unused")
public TimeZone getSystemTimezone() {
return TimeZone.getTimeZone(getVar(ConfVars.$TIMEZONE));
}
diff --git a/tajo-common/src/main/java/org/apache/tajo/datum/DateDatum.java b/tajo-common/src/main/java/org/apache/tajo/datum/DateDatum.java
index ac84e25..acc5408 100644
--- a/tajo-common/src/main/java/org/apache/tajo/datum/DateDatum.java
+++ b/tajo-common/src/main/java/org/apache/tajo/datum/DateDatum.java
@@ -102,29 +102,27 @@
@Override
public Datum plus(Datum datum) {
- switch(datum.type()) {
- case INT2:
- case INT4:
- case INT8:
- case FLOAT4:
- case FLOAT8: {
- TimeMeta tm = asTimeMeta();
- tm.plusDays(datum.asInt4());
- return new DateDatum(tm);
- }
- case INTERVAL:
- IntervalDatum interval = (IntervalDatum) datum;
- TimeMeta tm = asTimeMeta();
- tm.plusInterval(interval.months, interval.milliseconds);
- return new TimestampDatum(DateTimeUtil.toJulianTimestamp(tm));
- case TIME: {
- TimeMeta tm1 = asTimeMeta();
- TimeMeta tm2 = datum.asTimeMeta();
- tm1.plusTime(DateTimeUtil.toTime(tm2));
- return new TimestampDatum(DateTimeUtil.toJulianTimestamp(tm1));
- }
- default:
- throw new InvalidOperationException(datum.type());
+ switch (datum.type()) {
+ case INT2:
+ case INT4:
+ case INT8: {
+ TimeMeta tm = asTimeMeta();
+ tm.plusDays(datum.asInt4());
+ return new DateDatum(tm);
+ }
+ case INTERVAL:
+ IntervalDatum interval = (IntervalDatum) datum;
+ TimeMeta tm = asTimeMeta();
+ tm.plusInterval(interval.months, interval.milliseconds);
+ return new TimestampDatum(DateTimeUtil.toJulianTimestamp(tm));
+ case TIME: {
+ TimeMeta tm1 = asTimeMeta();
+ TimeMeta tm2 = datum.asTimeMeta();
+ tm1.plusTime(DateTimeUtil.toTime(tm2));
+ return new TimestampDatum(DateTimeUtil.toJulianTimestamp(tm1));
+ }
+ default:
+ throw new InvalidOperationException("operator does not exist: " + type() + " + " + datum.type());
}
}
@@ -133,9 +131,7 @@
switch(datum.type()) {
case INT2:
case INT4:
- case INT8:
- case FLOAT4:
- case FLOAT8: {
+ case INT8: {
TimeMeta tm = asTimeMeta();
tm.plusDays(0 - datum.asInt4());
return new DateDatum(tm);
@@ -157,7 +153,7 @@
return new Int4Datum(jdate - d.jdate);
}
default:
- throw new InvalidOperationException(datum.type());
+ throw new InvalidOperationException("operator does not exist: " + type() + " - " + datum.type());
}
}
@@ -207,7 +203,7 @@
} else if (datum.isNull()) {
return datum;
} else {
- throw new InvalidOperationException();
+ throw new InvalidOperationException(datum.type());
}
}
@@ -216,13 +212,7 @@
if (datum.type() == TajoDataTypes.Type.DATE) {
DateDatum another = (DateDatum) datum;
return Ints.compare(jdate, another.jdate);
- } else if (datum.type() == TajoDataTypes.Type.TIMESTAMP) {
- TimestampDatum another = (TimestampDatum) datum;
- TimeMeta myMeta, otherMeta;
- myMeta = asTimeMeta();
- otherMeta = another.asTimeMeta();
- return myMeta.compareTo(otherMeta);
- } else if (datum instanceof NullDatum || datum.isNull()) {
+ } else if (datum.isNull()) {
return -1;
} else {
throw new InvalidOperationException(datum.type());
diff --git a/tajo-common/src/main/java/org/apache/tajo/datum/Datum.java b/tajo-common/src/main/java/org/apache/tajo/datum/Datum.java
index e2173a8..dc61145 100644
--- a/tajo-common/src/main/java/org/apache/tajo/datum/Datum.java
+++ b/tajo-common/src/main/java/org/apache/tajo/datum/Datum.java
@@ -113,7 +113,7 @@
}
public byte[] asTextBytes() {
- return asChars().getBytes();
+ return asChars().getBytes(TextDatum.DEFAULT_CHARSET);
}
public TimeMeta asTimeMeta() {
diff --git a/tajo-common/src/main/java/org/apache/tajo/datum/DatumFactory.java b/tajo-common/src/main/java/org/apache/tajo/datum/DatumFactory.java
index e9ac0c5..aac877b 100644
--- a/tajo-common/src/main/java/org/apache/tajo/datum/DatumFactory.java
+++ b/tajo-common/src/main/java/org/apache/tajo/datum/DatumFactory.java
@@ -283,12 +283,6 @@
return new TimeDatum(DateTimeUtil.toJulianTime(timeStr));
}
- public static TimeDatum createTime(String timeStr, TimeZone tz) {
- TimeMeta tm = DateTimeUtil.decodeDateTime(timeStr);
- DateTimeUtil.toUTCTimezone(tm, tz);
- return new TimeDatum(DateTimeUtil.toTime(tm));
- }
-
public static TimestampDatum createTimestampDatumWithJavaMillis(long millis) {
return new TimestampDatum(DateTimeUtil.javaTimeToJulianTime(millis));
}
@@ -335,7 +329,7 @@
}
}
- public static TimeDatum createTime(Datum datum, @Nullable TimeZone tz) {
+ public static TimeDatum createTime(Datum datum) {
switch (datum.type()) {
case INT8:
return new TimeDatum(datum.asInt8());
@@ -343,9 +337,6 @@
case VARCHAR:
case TEXT:
TimeMeta tm = DateTimeFormat.parseDateTime(datum.asChars(), "HH24:MI:SS.MS");
- if (tz != null) {
- DateTimeUtil.toUTCTimezone(tm, tz);
- }
return new TimeDatum(DateTimeUtil.toTime(tm));
case TIME:
return (TimeDatum) datum;
@@ -362,6 +353,13 @@
return parseTimestamp(datum.asChars(), tz);
case TIMESTAMP:
return (TimestampDatum) datum;
+ case DATE: {
+ TimeMeta tm = datum.asTimeMeta();
+ if (tz != null) {
+ DateTimeUtil.toUTCTimezone(tm, tz);
+ }
+ return new TimestampDatum(DateTimeUtil.toJulianTimestamp(tm));
+ }
default:
throw new TajoRuntimeException(new InvalidValueForCastException(datum.type(), Type.TIMESTAMP));
}
@@ -436,21 +434,13 @@
return DatumFactory.createText(timestampDatum.asChars());
}
}
- case TIME: {
- TimeDatum timeDatum = (TimeDatum)operandDatum;
- if (tz != null) {
- return DatumFactory.createText(TimeDatum.asChars(operandDatum.asTimeMeta(), tz, false));
- } else {
- return DatumFactory.createText(timeDatum.asChars());
- }
- }
default:
return DatumFactory.createText(operandDatum.asTextBytes());
}
case DATE:
return DatumFactory.createDate(operandDatum);
case TIME:
- return DatumFactory.createTime(operandDatum, tz);
+ return DatumFactory.createTime(operandDatum);
case TIMESTAMP:
return DatumFactory.createTimestamp(operandDatum, tz);
case BLOB:
diff --git a/tajo-common/src/main/java/org/apache/tajo/datum/IntervalDatum.java b/tajo-common/src/main/java/org/apache/tajo/datum/IntervalDatum.java
index e797b87..44570f3 100644
--- a/tajo-common/src/main/java/org/apache/tajo/datum/IntervalDatum.java
+++ b/tajo-common/src/main/java/org/apache/tajo/datum/IntervalDatum.java
@@ -247,7 +247,7 @@
return new TimestampDatum(DateTimeUtil.toJulianTimestamp(tm));
}
default:
- throw new InvalidOperationException(datum.type());
+ throw new InvalidOperationException("operator does not exist: " + type() + " + " + datum.type());
}
}
@@ -257,7 +257,7 @@
IntervalDatum other = (IntervalDatum) datum;
return new IntervalDatum(months - other.months, milliseconds - other.milliseconds);
} else {
- throw new InvalidOperationException(datum.type());
+ throw new InvalidOperationException("operator does not exist: " + type() + " - " + datum.type());
}
}
@@ -274,7 +274,7 @@
double float8Val = datum.asFloat8();
return createIntervalDatum((double)months * float8Val, (double) milliseconds * float8Val);
default:
- throw new InvalidOperationException(datum.type());
+ throw new InvalidOperationException("operator does not exist: " + type() + " * " + datum.type());
}
}
@@ -297,7 +297,7 @@
}
return createIntervalDatum((double) months / paramValueF8, (double) milliseconds / paramValueF8);
default:
- throw new InvalidOperationException(datum.type());
+ throw new InvalidOperationException("operator does not exist: " + type() + " / " + datum.type());
}
}
@@ -380,6 +380,13 @@
sb.append(".").append(df2.format(millisecond));
}
}
+ } else {
+ sb.append(prefix)
+ .append(df.format(0))
+ .append(":")
+ .append(df.format(0))
+ .append(":")
+ .append(df.format(0));
}
}
@@ -406,7 +413,7 @@
} else if (datum.isNull()) {
return datum;
} else {
- throw new InvalidOperationException();
+ throw new InvalidOperationException(datum.type());
}
}
diff --git a/tajo-common/src/main/java/org/apache/tajo/datum/TimeDatum.java b/tajo-common/src/main/java/org/apache/tajo/datum/TimeDatum.java
index e70d7d5..57196d2 100644
--- a/tajo-common/src/main/java/org/apache/tajo/datum/TimeDatum.java
+++ b/tajo-common/src/main/java/org/apache/tajo/datum/TimeDatum.java
@@ -20,16 +20,14 @@
import com.google.common.primitives.Longs;
import org.apache.tajo.common.TajoDataTypes;
-import org.apache.tajo.exception.InvalidValueForCastException;
import org.apache.tajo.exception.InvalidOperationException;
+import org.apache.tajo.exception.InvalidValueForCastException;
import org.apache.tajo.exception.TajoRuntimeException;
import org.apache.tajo.util.Bytes;
import org.apache.tajo.util.datetime.DateTimeConstants.DateStyle;
import org.apache.tajo.util.datetime.DateTimeUtil;
import org.apache.tajo.util.datetime.TimeMeta;
-import java.util.TimeZone;
-
public class TimeDatum extends Datum {
public static final int SIZE = 8;
private final long time;
@@ -98,18 +96,6 @@
return DateTimeUtil.encodeTime(tm, DateStyle.ISO_DATES);
}
- public static String asChars(TimeMeta tm, TimeZone timeZone, boolean includeTimeZone) {
- DateTimeUtil.toUserTimezone(tm, timeZone);
- if (includeTimeZone) {
- tm.timeZone = timeZone.getRawOffset() / 1000;
- }
- return DateTimeUtil.encodeTime(tm, DateStyle.ISO_DATES);
- }
-
- public String toString(TimeZone timeZone, boolean includeTimeZone) {
- return asChars(asTimeMeta(), timeZone, includeTimeZone);
- }
-
@Override
public int size() {
return SIZE;
@@ -122,20 +108,27 @@
@Override
public Datum plus(Datum datum) {
- switch(datum.type()) {
- case INTERVAL:
- IntervalDatum interval = ((IntervalDatum)datum);
- TimeMeta tm = asTimeMeta();
- tm.plusInterval(interval.months, interval.milliseconds);
- return new TimeDatum(DateTimeUtil.toTime(tm));
- case DATE: {
- DateDatum dateDatum = (DateDatum) datum;
- TimeMeta dateTm = dateDatum.asTimeMeta();
- dateTm.plusTime(time);
- return new TimestampDatum(DateTimeUtil.toJulianTimestamp(dateTm));
- }
- default:
- throw new InvalidOperationException(datum.type());
+ switch (datum.type()) {
+ case INTERVAL: {
+ IntervalDatum interval = ((IntervalDatum) datum);
+ TimeMeta tm = asTimeMeta();
+ tm.plusInterval(interval.months, interval.milliseconds);
+ return new TimeDatum(DateTimeUtil.toTime(tm));
+ }
+ case DATE: {
+ DateDatum dateDatum = (DateDatum) datum;
+ TimeMeta dateTm = dateDatum.asTimeMeta();
+ dateTm.plusTime(time);
+ return new TimestampDatum(DateTimeUtil.toJulianTimestamp(dateTm));
+ }
+ case TIMESTAMP: {
+ TimestampDatum timestampDatum = (TimestampDatum) datum;
+ TimeMeta tm = timestampDatum.asTimeMeta();
+ tm.plusTime(time);
+ return new TimestampDatum(DateTimeUtil.toJulianTimestamp(tm));
+ }
+ default:
+ throw new InvalidOperationException("operator does not exist: " + type() + " + " + datum.type());
}
}
@@ -150,7 +143,7 @@
case TIME:
return new IntervalDatum((time - ((TimeDatum)datum).time)/1000);
default:
- throw new InvalidOperationException(datum.type());
+ throw new InvalidOperationException("operator does not exist: " + type() + " - " + datum.type());
}
}
@@ -170,7 +163,7 @@
if (datum.type() == TajoDataTypes.Type.TIME) {
TimeDatum another = (TimeDatum)datum;
return Longs.compare(time, another.time);
- } else if (datum instanceof NullDatum || datum.isNull()) {
+ } else if (datum.isNull()) {
return -1;
} else {
throw new InvalidOperationException(datum.type());
diff --git a/tajo-common/src/main/java/org/apache/tajo/datum/TimestampDatum.java b/tajo-common/src/main/java/org/apache/tajo/datum/TimestampDatum.java
index ac30278..782aab4 100644
--- a/tajo-common/src/main/java/org/apache/tajo/datum/TimestampDatum.java
+++ b/tajo-common/src/main/java/org/apache/tajo/datum/TimestampDatum.java
@@ -23,7 +23,6 @@
import org.apache.tajo.exception.InvalidOperationException;
import org.apache.tajo.util.Bytes;
import org.apache.tajo.util.datetime.DateTimeConstants.DateStyle;
-import org.apache.tajo.util.datetime.DateTimeFormat;
import org.apache.tajo.util.datetime.DateTimeUtil;
import org.apache.tajo.util.datetime.TimeMeta;
@@ -133,7 +132,7 @@
public static String asChars(TimeMeta tm, TimeZone timeZone, boolean includeTimeZone) {
DateTimeUtil.toUserTimezone(tm, timeZone);
if (includeTimeZone) {
- tm.timeZone = timeZone.getRawOffset() / 1000;
+ tm.timeZone = tm.getZonedOffset(DateTimeUtil.toJulianTimestamp(tm), timeZone) / 1000;
}
return DateTimeUtil.encodeDateTime(tm, DateStyle.ISO_DATES);
}
@@ -150,13 +149,7 @@
@Override
public String asChars() {
TimeMeta tm = asTimeMeta();
- return DateTimeUtil.encodeDateTime(tm, DateStyle.ISO_DATES);
- }
-
- public String toChars(String format) {
- TimeMeta tm = asTimeMeta();
-
- return DateTimeFormat.to_char(tm, format);
+ return asChars(tm, TimeZone.getDefault(), true);
}
@Override
@@ -190,11 +183,6 @@
if (datum.type() == TajoDataTypes.Type.TIMESTAMP) {
TimestampDatum another = (TimestampDatum) datum;
return Longs.compare(timestamp, another.timestamp);
- } else if (datum.type() == TajoDataTypes.Type.DATE) {
- TimeMeta myMeta, otherMeta;
- myMeta = asTimeMeta();
- otherMeta = datum.asTimeMeta();
- return myMeta.compareTo(otherMeta);
} else if (datum.isNull()) {
return -1;
} else {
@@ -214,28 +202,39 @@
@Override
public Datum plus(Datum datum) {
- if (datum.type() == TajoDataTypes.Type.INTERVAL) {
- IntervalDatum interval = (IntervalDatum)datum;
+ switch (datum.type()) {
+ case INTERVAL:
+ IntervalDatum interval = (IntervalDatum) datum;
TimeMeta tm = asTimeMeta();
tm.plusInterval(interval.months, interval.milliseconds);
return new TimestampDatum(DateTimeUtil.toJulianTimestamp(tm));
- } else {
- throw new InvalidOperationException(datum.type());
+ case TIME:
+ TimeMeta tm1 = asTimeMeta();
+ TimeMeta tm2 = datum.asTimeMeta();
+ tm1.plusTime(DateTimeUtil.toTime(tm2));
+ return new TimestampDatum(DateTimeUtil.toJulianTimestamp(tm1));
+ default:
+ throw new InvalidOperationException("operator does not exist: " + type() + " + " + datum.type());
}
}
@Override
public Datum minus(Datum datum) {
- switch(datum.type()) {
- case INTERVAL:
- IntervalDatum interval = (IntervalDatum)datum;
- TimeMeta tm = asTimeMeta();
- tm.plusInterval(-interval.months, -interval.milliseconds);
- return new TimestampDatum(DateTimeUtil.toJulianTimestamp(tm));
- case TIMESTAMP:
- return new IntervalDatum((timestamp - ((TimestampDatum)datum).timestamp) / 1000);
- default:
- throw new InvalidOperationException(datum.type());
+ switch (datum.type()) {
+ case INTERVAL:
+ IntervalDatum interval = (IntervalDatum) datum;
+ TimeMeta tm = asTimeMeta();
+ tm.plusInterval(-interval.months, -interval.milliseconds);
+ return new TimestampDatum(DateTimeUtil.toJulianTimestamp(tm));
+ case TIMESTAMP:
+ return new IntervalDatum((timestamp - ((TimestampDatum) datum).timestamp) / 1000);
+ case TIME:
+ TimeMeta tm1 = asTimeMeta();
+ TimeMeta tm2 = datum.asTimeMeta();
+ tm1.plusTime(0 - DateTimeUtil.toTime(tm2));
+ return new TimestampDatum(DateTimeUtil.toJulianTimestamp(tm1));
+ default:
+ throw new InvalidOperationException("operator does not exist: " + type() + " - " + datum.type());
}
}
diff --git a/tajo-common/src/main/java/org/apache/tajo/util/datetime/DateTimeUtil.java b/tajo-common/src/main/java/org/apache/tajo/util/datetime/DateTimeUtil.java
index 5a338d3..0bf58aa 100644
--- a/tajo-common/src/main/java/org/apache/tajo/util/datetime/DateTimeUtil.java
+++ b/tajo-common/src/main/java/org/apache/tajo/util/datetime/DateTimeUtil.java
@@ -18,6 +18,7 @@
package org.apache.tajo.util.datetime;
+import com.google.common.annotations.VisibleForTesting;
import org.apache.tajo.conf.TajoConf;
import org.apache.tajo.datum.Int8Datum;
import org.apache.tajo.exception.ValueOutOfRangeException;
@@ -26,6 +27,9 @@
import org.apache.tajo.util.datetime.DateTimeConstants.TokenField;
import javax.annotation.Nullable;
+import java.sql.Date;
+import java.sql.Time;
+import java.sql.Timestamp;
import java.util.TimeZone;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
@@ -356,6 +360,31 @@
return toTime(hour, min, sec, fsec)/DateTimeConstants.MSECS_PER_SEC;
}
+ public static Timestamp toJavaTimestamp(TimeMeta tm, @Nullable TimeZone tz) {
+ long javaTime = DateTimeUtil.julianTimeToJavaTime(DateTimeUtil.toJulianTimestamp(tm));
+
+ if (tz != null) {
+ int offset = tz.getOffset(javaTime) - TimeZone.getDefault().getOffset(javaTime);
+ return new Timestamp(javaTime + offset);
+ } else {
+ return new Timestamp(javaTime);
+ }
+ }
+
+ public static Time toJavaTime(TimeMeta tm, @Nullable TimeZone tz) {
+ if (tz != null) {
+ DateTimeUtil.toUserTimezone(tm, tz);
+ }
+ return new Time(tm.hours, tm.minutes, tm.secs);
+ }
+
+ public static Date toJavaDate(TimeMeta tm, @Nullable TimeZone tz) {
+ if (tz != null) {
+ DateTimeUtil.toUserTimezone(tm, tz);
+ }
+ return new Date(tm.years - 1900, tm.monthOfYear - 1 , tm.dayOfMonth);
+ }
+
/**
* Calculate julian timestamp.
* @param years
@@ -1891,7 +1920,7 @@
appendSecondsToEncodeOutput(sb, tm.secs, tm.fsecs, 6, true);
if (tm.timeZone != 0 && tm.timeZone != Integer.MAX_VALUE) {
- sb.append(getTimeZoneDisplayTime(tm.timeZone));
+ sb.append(getDisplayTimeZoneOffset(tm.timeZone));
}
if (tm.years <= 0) {
sb.append(" BC");
@@ -1938,7 +1967,7 @@
sb.append(String.format("%02d:%02d:", tm.hours, tm.minutes));
appendSecondsToEncodeOutput(sb, tm.secs, tm.fsecs, 6, true);
if (tm.timeZone != 0 && tm.timeZone != Integer.MAX_VALUE) {
- sb.append(getTimeZoneDisplayTime(tm.timeZone));
+ sb.append(getDisplayTimeZoneOffset(tm.timeZone));
}
break;
}
@@ -2077,18 +2106,19 @@
}
public static void toUserTimezone(TimeMeta tm, TimeZone timeZone) {
- tm.plusMillis(timeZone.getRawOffset());
+ tm.convertToLocalTime(timeZone);
}
public static void toUTCTimezone(TimeMeta tm, TimeZone timeZone) {
- tm.plusMillis(0 - timeZone.getRawOffset());
+ tm.convertToUTC(timeZone);
}
- public static String getTimeZoneDisplayTime(TimeZone timeZone) {
- return getTimeZoneDisplayTime(timeZone.getRawOffset() / 1000);
+ @VisibleForTesting
+ public static String getDisplayTimeZoneOffset(TimeZone timeZone, boolean dst) {
+ return getDisplayTimeZoneOffset((timeZone.getRawOffset() + (dst ? timeZone.getDSTSavings() : 0)) / 1000);
}
- public static String getTimeZoneDisplayTime(int totalSecs) {
+ public static String getDisplayTimeZoneOffset(int totalSecs) {
if (totalSecs == 0) {
return "";
}
diff --git a/tajo-common/src/main/java/org/apache/tajo/util/datetime/TimeMeta.java b/tajo-common/src/main/java/org/apache/tajo/util/datetime/TimeMeta.java
index 3349f43..7202def 100644
--- a/tajo-common/src/main/java/org/apache/tajo/util/datetime/TimeMeta.java
+++ b/tajo-common/src/main/java/org/apache/tajo/util/datetime/TimeMeta.java
@@ -20,6 +20,8 @@
import org.apache.tajo.util.datetime.DateTimeConstants.DateStyle;
+import java.util.TimeZone;
+
public class TimeMeta implements Comparable<TimeMeta> {
public int fsecs; // 1/1,000,000 secs
public int secs;
@@ -130,6 +132,24 @@
plusTime(millis * 1000);
}
+ public void convertToUTC(TimeZone zone) {
+ long timestamp = DateTimeUtil.toJulianTimestamp(this);
+
+ timestamp -= (getZonedOffset(timestamp, zone) * 1000L);
+ DateTimeUtil.toJulianTimeMeta(timestamp, this);
+ }
+
+ public void convertToLocalTime(TimeZone zone) {
+ long timestamp = DateTimeUtil.toJulianTimestamp(this);
+
+ timestamp += (getZonedOffset(timestamp, zone) * 1000L);
+ DateTimeUtil.toJulianTimeMeta(timestamp, this);
+ }
+
+ public int getZonedOffset(long julianTimestamp, TimeZone zone) {
+ return zone.getOffset(DateTimeUtil.julianTimeToJavaTime(julianTimestamp));
+ }
+
public void plusTime(long time) {
long timestamp = DateTimeUtil.toJulianTimestamp(this);
timestamp += time;
diff --git a/tajo-common/src/test/java/org/apache/tajo/datum/TestDateDatum.java b/tajo-common/src/test/java/org/apache/tajo/datum/TestDateDatum.java
index 2b787f5..c7014a6 100644
--- a/tajo-common/src/test/java/org/apache/tajo/datum/TestDateDatum.java
+++ b/tajo-common/src/test/java/org/apache/tajo/datum/TestDateDatum.java
@@ -117,9 +117,5 @@
assertThat(theday.compareTo(thedaybefore) > 0, is(true));
assertThat(thedaybefore.compareTo(theday) > 0, is(false));
-
- TimestampDatum timestamp = DatumFactory.createTimestamp("2014-11-12 15:00:00.68");
-
- assertThat(timestamp.compareTo(theday) > 0, is(true));
}
}
diff --git a/tajo-common/src/test/java/org/apache/tajo/datum/TestIntervalDatum.java b/tajo-common/src/test/java/org/apache/tajo/datum/TestIntervalDatum.java
index 3175c51..b319c9c 100644
--- a/tajo-common/src/test/java/org/apache/tajo/datum/TestIntervalDatum.java
+++ b/tajo-common/src/test/java/org/apache/tajo/datum/TestIntervalDatum.java
@@ -64,8 +64,7 @@
// date '2001-09-28' + integer '7' ==> date '2001-10-05'
Datum datum = DatumFactory.createDate(2001, 9, 28);
- Datum[] datums = new Datum[]{new Int2Datum((short) 7), new Int4Datum(7), new Int8Datum(7),
- new Float4Datum(7.0f), new Float8Datum(7.0f)};
+ Datum[] datums = new Datum[]{new Int2Datum((short) 7), new Int4Datum(7), new Int8Datum(7)};
for (int i = 0; i < datums.length; i++) {
Datum result = datum.plus(datums[i]);
@@ -78,20 +77,20 @@
datum = DatumFactory.createDate(2001, 9, 28);
Datum result = datum.plus(new IntervalDatum(60 * 60 * 1000));
assertEquals(TajoDataTypes.Type.TIMESTAMP, result.type());
- assertEquals("2001-09-28 01:00:00", result.asChars());
+ assertEquals(DatumFactory.parseTimestamp("2001-09-28 01:00:00", null), result);
// interval '1 hour' + date '2001-09-28' ==> timestamp '2001-09-28 01:00:00'
datum = new IntervalDatum(60 * 60 * 1000);
result = datum.plus(DatumFactory.createDate(2001, 9, 28));
assertEquals(TajoDataTypes.Type.TIMESTAMP, result.type());
- assertEquals("2001-09-28 01:00:00", result.asChars());
+ assertEquals(DatumFactory.parseTimestamp("2001-09-28 01:00:00", null), result);
// date '2001-09-28' + time '03:00' ==> timestamp '2001-09-28 03:00:00'
datum = DatumFactory.createDate(2001, 9, 28);
TimeDatum time = new TimeDatum(DateTimeUtil.toTime(3, 0, 0, 0));
result = datum.plus(time);
assertEquals(TajoDataTypes.Type.TIMESTAMP, result.type());
- assertEquals("2001-09-28 03:00:00", result.asChars());
+ assertEquals(DatumFactory.parseTimestamp("2001-09-28 03:00:00", null), result);
// interval '1 day' + interval '1 hour' interval '1 day 01:00:00'
datum = new IntervalDatum(IntervalDatum.DAY_MILLIS);
@@ -103,7 +102,7 @@
datum = new TimestampDatum(DateTimeUtil.toJulianTimestamp(2001, 9, 28, 1, 0, 0, 0));
result = datum.plus(new IntervalDatum(23 * 60 * 60 * 1000));
assertEquals(TajoDataTypes.Type.TIMESTAMP, result.type());
- assertEquals("2001-09-29 00:00:00", result.asChars());
+ assertEquals(DatumFactory.parseTimestamp("2001-09-29 00:00:00", null), result);
// time '01:00' + interval '3 hours' ==> time '04:00:00'
datum = new TimeDatum(DateTimeUtil.toTime(1, 0, 0, 0));
@@ -132,14 +131,14 @@
datum = DatumFactory.createDate(2001, 9, 28);
result = datum.minus(new IntervalDatum(1 * 60 * 60 * 1000));
assertEquals(TajoDataTypes.Type.TIMESTAMP, result.type());
- assertEquals("2001-09-27 23:00:00", result.asChars());
+ assertEquals(DatumFactory.parseTimestamp("2001-09-27 23:00:00", null), result);
// date '2001-09-28' - interval '1 day 1 hour' ==> timestamp '2001-09-26 23:00:00'
// In this case all datums are UTC
datum = DatumFactory.createDate(2001, 9, 28);
result = datum.minus(new IntervalDatum(IntervalDatum.DAY_MILLIS + 1 * 60 * 60 * 1000));
assertEquals(TajoDataTypes.Type.TIMESTAMP, result.type());
- assertEquals("2001-09-26 23:00:00", result.asChars());
+ assertEquals(DatumFactory.parseTimestamp("2001-09-26 23:00:00", null), result);
// time '05:00' - time '03:00' ==> interval '02:00:00'
datum = new TimeDatum(DateTimeUtil.toTime(5, 0, 0, 0));
@@ -158,7 +157,7 @@
datum = new TimestampDatum(DateTimeUtil.toJulianTimestamp(2001, 9, 28, 23, 0, 0, 0));
result = datum.minus(new IntervalDatum(23 * 60 * 60 * 1000));
assertEquals(TajoDataTypes.Type.TIMESTAMP, result.type());
- assertEquals("2001-09-28 00:00:00", result.asChars());
+ assertEquals(DatumFactory.parseTimestamp("2001-09-28 00:00:00", null), result);
// interval '1 day' - interval '1 hour' ==> interval '1 day -01:00:00'
datum = new IntervalDatum(IntervalDatum.DAY_MILLIS);
@@ -204,6 +203,6 @@
datum = new TimestampDatum(DateTimeUtil.toJulianTimestamp(2001, 8, 31, 1, 0, 0, 0));
result = datum.plus(new IntervalDatum(1, 0));
assertEquals(TajoDataTypes.Type.TIMESTAMP, result.type());
- assertEquals("2001-09-30 01:00:00", result.asChars());
+ assertEquals(DatumFactory.parseTimestamp("2001-09-30 01:00:00", null), result);
}
}
diff --git a/tajo-common/src/test/java/org/apache/tajo/datum/TestTimeDatum.java b/tajo-common/src/test/java/org/apache/tajo/datum/TestTimeDatum.java
index 457ff41..7058b52 100644
--- a/tajo-common/src/test/java/org/apache/tajo/datum/TestTimeDatum.java
+++ b/tajo-common/src/test/java/org/apache/tajo/datum/TestTimeDatum.java
@@ -18,8 +18,8 @@
package org.apache.tajo.datum;
-import org.apache.tajo.common.TajoDataTypes.Type;
import org.apache.tajo.common.TajoDataTypes.DataType;
+import org.apache.tajo.common.TajoDataTypes.Type;
import org.apache.tajo.exception.TajoRuntimeException;
import org.apache.tajo.json.CommonGsonHelper;
import org.junit.Test;
diff --git a/tajo-common/src/test/java/org/apache/tajo/datum/TestTimestampDatum.java b/tajo-common/src/test/java/org/apache/tajo/datum/TestTimestampDatum.java
index 68b34a6..2bbb92d 100644
--- a/tajo-common/src/test/java/org/apache/tajo/datum/TestTimestampDatum.java
+++ b/tajo-common/src/test/java/org/apache/tajo/datum/TestTimestampDatum.java
@@ -92,7 +92,7 @@
public void testAsText2() {
// TAJO-1366
TimestampDatum datum = DatumFactory.createTimestamp("Mon Nov 03 00:03:00 +0000 2014");
- assertEquals("2014-11-03 00:03:00", datum.asChars());
+ assertEquals("2014-11-03 00:03:00", TimestampDatum.asChars(datum.asTimeMeta(), TimeZone.getTimeZone("+0000"), true));
}
@Test
@@ -120,7 +120,7 @@
@Test
public final void testTimeZone() {
TimestampDatum datum = new TimestampDatum(DateTimeUtil.toJulianTimestamp(2014, 5, 1, 15, 20, 30, 0));
- assertEquals("2014-05-01 15:20:30", datum.asChars());
+ assertEquals("2014-05-01 15:20:30", TimestampDatum.asChars(datum.asTimeMeta(), TimeZone.getTimeZone("GMT"), true));
assertEquals("2014-05-02 00:20:30+09", TimestampDatum.asChars(datum.asTimeMeta(), TimeZone.getTimeZone("GMT+9"), true));
}
@@ -195,10 +195,6 @@
assertThat(theday.compareTo(thedaybefore) > 0, is(true));
assertThat(thedaybefore.compareTo(theday) > 0, is(false));
-
- DateDatum date = DatumFactory.createDate("2014-11-12");
-
- assertThat(theday.compareTo(date) > 0, is(true));
}
@Test
diff --git a/tajo-common/src/test/java/org/apache/tajo/util/TestDateTimeUtil.java b/tajo-common/src/test/java/org/apache/tajo/util/TestDateTimeUtil.java
index ae4a517..75f3ee7 100644
--- a/tajo-common/src/test/java/org/apache/tajo/util/TestDateTimeUtil.java
+++ b/tajo-common/src/test/java/org/apache/tajo/util/TestDateTimeUtil.java
@@ -384,11 +384,13 @@
@Test
public void testGetTimeZoneDisplayTime() {
- assertEquals("", DateTimeUtil.getTimeZoneDisplayTime(TimeZone.getTimeZone("GMT")));
- assertEquals("+09", DateTimeUtil.getTimeZoneDisplayTime(TimeZone.getTimeZone("GMT+9")));
- assertEquals("+09:10", DateTimeUtil.getTimeZoneDisplayTime(TimeZone.getTimeZone("GMT+9:10")));
- assertEquals("-09", DateTimeUtil.getTimeZoneDisplayTime(TimeZone.getTimeZone("GMT-9")));
- assertEquals("-09:10", DateTimeUtil.getTimeZoneDisplayTime(TimeZone.getTimeZone("GMT-9:10")));
+ assertEquals("", DateTimeUtil.getDisplayTimeZoneOffset(TimeZone.getTimeZone("GMT"), false));
+ assertEquals("+09", DateTimeUtil.getDisplayTimeZoneOffset(TimeZone.getTimeZone("GMT+9"), false));
+ assertEquals("+09:10", DateTimeUtil.getDisplayTimeZoneOffset(TimeZone.getTimeZone("GMT+9:10"), false));
+ assertEquals("-09", DateTimeUtil.getDisplayTimeZoneOffset(TimeZone.getTimeZone("GMT-9"), false));
+ assertEquals("-09:10", DateTimeUtil.getDisplayTimeZoneOffset(TimeZone.getTimeZone("GMT-9:10"), false));
+ assertEquals("-07", DateTimeUtil.getDisplayTimeZoneOffset(TimeZone.getTimeZone("America/Los_Angeles"), true));
+ assertEquals("-08", DateTimeUtil.getDisplayTimeZoneOffset(TimeZone.getTimeZone("America/Los_Angeles"), false));
}
@Test
diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/cli/tools/TestDDLBuilder.java b/tajo-core-tests/src/test/java/org/apache/tajo/cli/tools/TestDDLBuilder.java
index 8790a38..4bef226 100644
--- a/tajo-core-tests/src/test/java/org/apache/tajo/cli/tools/TestDDLBuilder.java
+++ b/tajo-core-tests/src/test/java/org/apache/tajo/cli/tools/TestDDLBuilder.java
@@ -20,14 +20,18 @@
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.compress.GzipCodec;
+import org.apache.tajo.BuiltinStorages;
import org.apache.tajo.catalog.*;
import org.apache.tajo.catalog.partition.PartitionMethodDesc;
import org.apache.tajo.catalog.proto.CatalogProtos;
import org.apache.tajo.common.TajoDataTypes;
+import org.apache.tajo.conf.TajoConf;
import org.apache.tajo.storage.StorageConstants;
import org.apache.tajo.util.JavaResourceUtil;
import org.junit.Test;
+import java.util.TimeZone;
+
import static org.junit.Assert.*;
@@ -35,6 +39,7 @@
private static final Schema schema1;
private static final TableMeta meta1;
private static final PartitionMethodDesc partitionMethod1;
+ private static final TajoConf conf;
static {
schema1 = SchemaBuilder.builder()
@@ -42,8 +47,9 @@
.add("addr", TajoDataTypes.Type.TEXT)
.build();
- meta1 = CatalogUtil.newTableMeta("TEXT");
- meta1.putProperty(StorageConstants.TEXT_DELIMITER, StorageConstants.DEFAULT_FIELD_DELIMITER);
+ conf = new TajoConf();
+ conf.setSystemTimezone(TimeZone.getTimeZone("Asia/Seoul"));
+ meta1 = CatalogUtil.newTableMeta(BuiltinStorages.TEXT, conf);
meta1.putProperty(StorageConstants.COMPRESSION_CODEC, GzipCodec.class.getName());
Schema expressionSchema = SchemaBuilder.builder()
diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/cli/tools/TestTajoDump.java b/tajo-core-tests/src/test/java/org/apache/tajo/cli/tools/TestTajoDump.java
index 937b75c..57a6aa3 100644
--- a/tajo-core-tests/src/test/java/org/apache/tajo/cli/tools/TestTajoDump.java
+++ b/tajo-core-tests/src/test/java/org/apache/tajo/cli/tools/TestTajoDump.java
@@ -49,7 +49,9 @@
TajoDump.dump(client, userInfo, getCurrentDatabase(), false, false, false, printWriter);
printWriter.flush();
printWriter.close();
- assertStrings(new String(bos.toByteArray()));
+
+ assertOutputResult("testDump1.result", new String(bos.toByteArray()), new String[]{"${table.timezone}"},
+ new String[]{testingCluster.getConfiguration().getSystemTimezone().getID()});
bos.close();
} finally {
executeString("DROP TABLE \"" + getCurrentDatabase() + "\".\"TableName1\"");
@@ -70,7 +72,9 @@
TajoDump.dump(client, userInfo, getCurrentDatabase(), false, false, false, printWriter);
printWriter.flush();
printWriter.close();
- assertStrings(new String(bos.toByteArray()));
+
+ assertOutputResult("testDump2.result", new String(bos.toByteArray()), new String[]{"${table.timezone}"},
+ new String[]{testingCluster.getConfiguration().getSystemTimezone().getID()});
bos.close();
} finally {
executeString("DROP TABLE \"" + getCurrentDatabase() + "\".\"TableName2\"");
@@ -95,8 +99,10 @@
printWriter.flush();
printWriter.close();
- assertOutputResult("testDump3.result", new String(bos.toByteArray()), new String[]{"${index.path}"},
- new String[]{TablespaceManager.getDefault().getTableUri(getCurrentDatabase(), "test_idx").toString()});
+ assertOutputResult("testDump3.result", new String(bos.toByteArray()),
+ new String[]{"${index.path}", "${table.timezone}"},
+ new String[]{TablespaceManager.getDefault().getTableUri(getCurrentDatabase(), "test_idx").toString(),
+ testingCluster.getConfiguration().getSystemTimezone().getID()});
bos.close();
} finally {
executeString("DROP INDEX test_idx");
@@ -131,13 +137,12 @@
printWriter.flush();
printWriter.close();
- String[] paramValues = new String[] {
- TablespaceManager.getDefault().getTableUri(getCurrentDatabase(), "TableName3").toString()
- , TablespaceManager.getDefault().getTableUri(getCurrentDatabase(), "TableName4").toString()
- };
+ assertOutputResult("testPartitionsDump.result", new String(bos.toByteArray()),
+ new String[]{"${partition.path1}", "${partition.path2}", "${table.timezone}"},
+ new String[]{TablespaceManager.getDefault().getTableUri(getCurrentDatabase(), "TableName3").toString(),
+ TablespaceManager.getDefault().getTableUri(getCurrentDatabase(), "TableName4").toString(),
+ testingCluster.getConfiguration().getSystemTimezone().getID()});
- assertOutputResult("testPartitionsDump.result", new String(bos.toByteArray())
- , new String[]{"${partition.path1}", "${partition.path2}"}, paramValues);
bos.close();
} finally {
executeString("DROP TABLE \"" + getCurrentDatabase() + "\".\"TableName3\"");
diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/cli/tsql/TestTajoCli.java b/tajo-core-tests/src/test/java/org/apache/tajo/cli/tsql/TestTajoCli.java
index c210a24..c148298 100644
--- a/tajo-core-tests/src/test/java/org/apache/tajo/cli/tsql/TestTajoCli.java
+++ b/tajo-core-tests/src/test/java/org/apache/tajo/cli/tsql/TestTajoCli.java
@@ -251,8 +251,9 @@
String consoleResult = new String(out.toByteArray());
if (!cluster.isHiveCatalogStoreRunning()) {
- assertOutputResult(resultFileName, consoleResult, new String[]{"${table.path}"},
- new String[]{TablespaceManager.getDefault().getTableUri("default", tableName).toString()});
+ assertOutputResult(resultFileName, consoleResult, new String[]{"${table.timezone}", "${table.path}"},
+ new String[]{cluster.getConfiguration().getSystemTimezone().getID(),
+ TablespaceManager.getDefault().getTableUri("default", tableName).toString()});
}
}
@@ -485,10 +486,12 @@
tajoCli.executeMetaCommand("\\set TIMEZONE GMT+0");
tajoCli.executeScript("create table " + tableName + " (col1 TIMESTAMP)");
tajoCli.executeScript("insert into " + tableName + " select to_timestamp(0)");
+ out.reset();
+
tajoCli.executeScript("select * from " + tableName);
String consoleResult = new String(out.toByteArray());
tajoCli.executeScript("DROP TABLE " + tableName + " PURGE");
- assertTrue(consoleResult.contains("1970-01-01 00:00:00"));
+ assertEquals("1970-01-01 00:00:00", consoleResult.split("\n")[2]);
}
@Test
@@ -497,10 +500,12 @@
tajoCli.executeMetaCommand("\\set TIMEZONE GMT+1");
tajoCli.executeScript("create table " + tableName + " (col1 TIMESTAMP)");
tajoCli.executeScript("insert into " + tableName + " select to_timestamp(0)");
+ out.reset();
+
tajoCli.executeScript("select * from " + tableName);
String consoleResult = new String(out.toByteArray());
tajoCli.executeScript("DROP TABLE " + tableName + " PURGE");
- assertTrue(consoleResult.contains("1970-01-01 00:00:00"));
+ assertEquals("1970-01-01 01:00:00", consoleResult.split("\n")[2]);
}
@Test(timeout = 3000)
diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/codegen/TestEvalCodeGenerator.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/codegen/TestEvalCodeGenerator.java
index adbfd48..27ad24b 100644
--- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/codegen/TestEvalCodeGenerator.java
+++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/codegen/TestEvalCodeGenerator.java
@@ -82,28 +82,28 @@
.add("nullable", TajoDataTypes.Type.NULL_TYPE)
.build();
- testEval(schema, "table1", ",1,2,3,4.5,6.5,F6,abc,abc,t", "select col0 is null from table1;", new String [] {"t"});
- testEval(schema, "table1", "0,,2,3,4.5,6.5,F6,abc,abc,t,", "select col1 is null from table1;", new String [] {"t"});
- testEval(schema, "table1", "0,1,,3,4.5,6.5,F6,abc,abc,t,", "select col2 is null from table1;", new String [] {"t"});
- testEval(schema, "table1", "0,1,2,,4.5,6.5,F6,abc,abc,t,", "select col3 is null from table1;", new String [] {"t"});
- testEval(schema, "table1", "0,1,2,3,,6.5,F6,abc,abc,t,", "select col4 is null from table1;", new String [] {"t"});
- testEval(schema, "table1", "0,1,2,3,4.5,,F6,abc,abc,t,", "select col5 is null from table1;", new String [] {"t"});
- testEval(schema, "table1", "0,1,2,3,4.5,6.5,,abc,abc,t,", "select col6 is null from table1;", new String [] {"t"});
- testEval(schema, "table1", "0,1,2,3,4.5,6.5,F6,,abc,t,", "select col7 is null from table1;", new String[]{"t"});
- testEval(schema, "table1", "0,1,2,3,4.5,6.5,F6,abc,,t,", "select col8 is null from table1;", new String [] {"t"});
- testEval(schema, "table1", "0,1,2,3,4.5,6.5,F6,abc,abc,,", "select col9 is null from table1;", new String [] {"t"});
+ testEval(schema, "table1", "\\NULL,1,2,3,4.5,6.5,F6,abc,abc,t", "select col0 is null from table1;", new String [] {"t"});
+ testEval(schema, "table1", "0,\\NULL,2,3,4.5,6.5,F6,abc,abc,t,", "select col1 is null from table1;", new String [] {"t"});
+ testEval(schema, "table1", "0,1,\\NULL,3,4.5,6.5,F6,abc,abc,t,", "select col2 is null from table1;", new String [] {"t"});
+ testEval(schema, "table1", "0,1,2,\\NULL,4.5,6.5,F6,abc,abc,t,", "select col3 is null from table1;", new String [] {"t"});
+ testEval(schema, "table1", "0,1,2,3,\\NULL,6.5,F6,abc,abc,t,", "select col4 is null from table1;", new String [] {"t"});
+ testEval(schema, "table1", "0,1,2,3,4.5,\\NULL,F6,abc,abc,t,", "select col5 is null from table1;", new String [] {"t"});
+ testEval(schema, "table1", "0,1,2,3,4.5,6.5,\\NULL,abc,abc,t,", "select col6 is null from table1;", new String [] {"t"});
+ testEval(schema, "table1", "0,1,2,3,4.5,6.5,F6,\\NULL,abc,t,", "select col7 is null from table1;", new String[]{"t"});
+ testEval(schema, "table1", "0,1,2,3,4.5,6.5,F6,abc,\\NULL,t,", "select col8 is null from table1;", new String [] {"t"});
+ testEval(schema, "table1", "0,1,2,3,4.5,6.5,F6,abc,abc,\\NULL,", "select col9 is null from table1;", new String [] {"t"});
testEval(schema, "table1", "0,1,2,3,4.5,6.5,F6,abc,abc,t,", "select nullable is null from table1;", new String [] {"t"});
- testEval(schema, "table1", ",1,2,3,4.5,6.5,F6,abc,abc,t", "select col0 is not null from table1;", new String [] {"f"});
- testEval(schema, "table1", "0,,2,3,4.5,6.5,F6,abc,abc,t,", "select col1 is not null from table1;", new String [] {"f"});
- testEval(schema, "table1", "0,1,,3,4.5,6.5,F6,abc,abc,t,", "select col2 is not null from table1;", new String [] {"f"});
- testEval(schema, "table1", "0,1,2,,4.5,6.5,F6,abc,abc,t,", "select col3 is not null from table1;", new String [] {"f"});
- testEval(schema, "table1", "0,1,2,3,,6.5,F6,abc,abc,t,", "select col4 is not null from table1;", new String [] {"f"});
- testEval(schema, "table1", "0,1,2,3,4.5,,F6,abc,abc,t,", "select col5 is not null from table1;", new String [] {"f"});
- testEval(schema, "table1", "0,1,2,3,4.5,6.5,,abc,abc,t,", "select col6 is not null from table1;", new String [] {"f"});
- testEval(schema, "table1", "0,1,2,3,4.5,6.5,F6,,abc,t,", "select col7 is not null from table1;", new String [] {"f"});
- testEval(schema, "table1", "0,1,2,3,4.5,6.5,F6,abc,,t,", "select col8 is not null from table1;", new String [] {"f"});
- testEval(schema, "table1", "0,1,2,3,4.5,6.5,F6,abc,abc,,", "select col9 is not null from table1;", new String [] {"f"});
+ testEval(schema, "table1", "\\NULL,1,2,3,4.5,6.5,F6,abc,abc,t", "select col0 is not null from table1;", new String [] {"f"});
+ testEval(schema, "table1", "0,\\NULL,2,3,4.5,6.5,F6,abc,abc,t,", "select col1 is not null from table1;", new String [] {"f"});
+ testEval(schema, "table1", "0,1,\\NULL,3,4.5,6.5,F6,abc,abc,t,", "select col2 is not null from table1;", new String [] {"f"});
+ testEval(schema, "table1", "0,1,2,\\NULL,4.5,6.5,F6,abc,abc,t,", "select col3 is not null from table1;", new String [] {"f"});
+ testEval(schema, "table1", "0,1,2,3,\\NULL,6.5,F6,abc,abc,t,", "select col4 is not null from table1;", new String [] {"f"});
+ testEval(schema, "table1", "0,1,2,3,4.5,\\NULL,F6,abc,abc,t,", "select col5 is not null from table1;", new String [] {"f"});
+ testEval(schema, "table1", "0,1,2,3,4.5,6.5,\\NULL,abc,abc,t,", "select col6 is not null from table1;", new String [] {"f"});
+ testEval(schema, "table1", "0,1,2,3,4.5,6.5,F6,\\NULL,abc,t,", "select col7 is not null from table1;", new String [] {"f"});
+ testEval(schema, "table1", "0,1,2,3,4.5,6.5,F6,abc,\\NULL,t,", "select col8 is not null from table1;", new String [] {"f"});
+ testEval(schema, "table1", "0,1,2,3,4.5,6.5,F6,abc,abc,\\NULL,", "select col9 is not null from table1;", new String [] {"f"});
testEval(schema, "table1", "0,1,2,3,4.5,6.5,F6,abc,abc,t,", "select nullable is not null from table1;", new String [] {"f"});
}
diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java
index 00fa590..38911c4 100644
--- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java
+++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java
@@ -18,10 +18,9 @@
package org.apache.tajo.engine.eval;
-import org.apache.tajo.LocalTajoTestingUtility;
-import org.apache.tajo.OverridableConf;
-import org.apache.tajo.SessionVars;
-import org.apache.tajo.TajoTestingCluster;
+import io.netty.buffer.Unpooled;
+import org.apache.commons.lang.StringEscapeUtils;
+import org.apache.tajo.*;
import org.apache.tajo.algebra.Expr;
import org.apache.tajo.catalog.*;
import org.apache.tajo.cli.tsql.InvalidStatementException;
@@ -52,13 +51,10 @@
import org.apache.tajo.plan.verifier.LogicalPlanVerifier;
import org.apache.tajo.plan.verifier.PreLogicalPlanVerifier;
import org.apache.tajo.plan.verifier.VerificationState;
-import org.apache.tajo.storage.LazyTuple;
-import org.apache.tajo.storage.TablespaceManager;
-import org.apache.tajo.storage.Tuple;
-import org.apache.tajo.storage.VTuple;
-import org.apache.tajo.util.BytesUtils;
+import org.apache.tajo.storage.*;
+import org.apache.tajo.storage.text.CSVLineSerDe;
+import org.apache.tajo.storage.text.TextLineDeserializer;
import org.apache.tajo.util.CommonTestingUtil;
-import org.apache.tajo.util.KeyValueSet;
import org.apache.tajo.util.datetime.DateTimeUtil;
import org.junit.AfterClass;
import org.junit.BeforeClass;
@@ -85,7 +81,7 @@
private static LogicalPlanVerifier annotatedPlanVerifier;
public static String getUserTimeZoneDisplay(TimeZone tz) {
- return DateTimeUtil.getTimeZoneDisplayTime(tz);
+ return DateTimeUtil.getDisplayTimeZoneOffset(tz, false);
}
public ExprTestBase() {
@@ -217,60 +213,55 @@
if (context == null) {
queryContext = LocalTajoTestingUtility.createDummyContext(conf);
} else {
- queryContext = LocalTajoTestingUtility.createDummyContext(conf);
+ queryContext = LocalTajoTestingUtility.createDummyContext(context.getConf());
queryContext.putAll(context);
}
- String timezoneId = queryContext.get(SessionVars.TIMEZONE);
- TimeZone timeZone = TimeZone.getTimeZone(timezoneId);
-
- LazyTuple lazyTuple;
VTuple vtuple = null;
String qualifiedTableName =
CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME,
tableName != null ? CatalogUtil.normalizeIdentifier(tableName) : null);
Schema inputSchema = null;
+
+
+ TableMeta meta = CatalogUtil.newTableMeta(BuiltinStorages.TEXT, queryContext.getConf());
+ meta.putProperty(StorageConstants.TEXT_DELIMITER, StringEscapeUtils.escapeJava(delimiter+""));
+ meta.putProperty(StorageConstants.TEXT_NULL, StringEscapeUtils.escapeJava("\\NULL"));
+
+ String timezoneId = queryContext.get(SessionVars.TIMEZONE);
+ TimeZone timeZone = TimeZone.getTimeZone(timezoneId);
+
if (schema != null) {
inputSchema = SchemaUtil.clone(schema);
inputSchema.setQualifier(qualifiedTableName);
- int targetIdx [] = new int[inputSchema.size()];
- for (int i = 0; i < targetIdx.length; i++) {
- targetIdx[i] = i;
- }
-
- byte[][] tokens = BytesUtils.splitPreserveAllTokens(
- csvTuple.getBytes(), delimiter, targetIdx, inputSchema.size());
- lazyTuple = new LazyTuple(inputSchema, tokens,0);
- vtuple = new VTuple(inputSchema.size());
- for (int i = 0; i < inputSchema.size(); i++) {
-
- // If null value occurs, null datum is manually inserted to an input tuple.
- boolean nullDatum;
- Datum datum = lazyTuple.get(i);
- nullDatum = (datum instanceof TextDatum || datum instanceof CharDatum);
- nullDatum = nullDatum &&
- datum.asChars().equals("") || datum.asChars().equals(queryContext.get(SessionVars.NULL_CHAR));
- nullDatum |= datum.isNull();
-
- if (nullDatum) {
- vtuple.put(i, NullDatum.get());
- } else {
- vtuple.put(i, lazyTuple.get(i));
- }
- }
try {
- cat.createTable(new TableDesc(qualifiedTableName, inputSchema,"TEXT",
- new KeyValueSet(), CommonTestingUtil.getTestDir().toUri()));
+ cat.createTable(CatalogUtil.newTableDesc(
+ qualifiedTableName, inputSchema, meta, CommonTestingUtil.getTestDir()));
} catch (IOException e) {
throw new TajoInternalError(e);
}
+
+ CSVLineSerDe serDe = new CSVLineSerDe();
+ TextLineDeserializer deserializer = serDe.createDeserializer(inputSchema, meta, inputSchema.toArray());
+ deserializer.init();
+
+ vtuple = new VTuple(inputSchema.size());
+
+ try {
+ deserializer.deserialize(Unpooled.wrappedBuffer(csvTuple.getBytes()), vtuple);
+ } catch (Exception e) {
+ throw new TajoInternalError(e);
+ } finally {
+ deserializer.release();
+ }
}
List<Target> targets;
TajoClassLoader classLoader = new TajoClassLoader();
EvalContext evalContext = new EvalContext();
+ evalContext.setTimeZone(timeZone);
try {
if (needPythonFileCopy()) {
@@ -306,8 +297,6 @@
String outTupleAsChars;
if (outTuple.type(i) == Type.TIMESTAMP) {
outTupleAsChars = TimestampDatum.asChars(outTuple.getTimeDate(i), timeZone, false);
- } else if (outTuple.type(i) == Type.TIME) {
- outTupleAsChars = TimeDatum.asChars(outTuple.getTimeDate(i), timeZone, false);
} else {
outTupleAsChars = outTuple.asDatum(i).toString();
}
diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/eval/TestEvalTreeUtil.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/eval/TestEvalTreeUtil.java
index 2846f22..a0cb924 100644
--- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/eval/TestEvalTreeUtil.java
+++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/eval/TestEvalTreeUtil.java
@@ -19,6 +19,7 @@
package org.apache.tajo.engine.eval;
import com.google.common.collect.Sets;
+import org.apache.tajo.BuiltinStorages;
import org.apache.tajo.LocalTajoTestingUtility;
import org.apache.tajo.TajoConstants;
import org.apache.tajo.TajoTestingCluster;
@@ -103,7 +104,7 @@
.add("age", TajoDataTypes.Type.INT4)
.build();
- TableMeta meta = CatalogUtil.newTableMeta("TEXT");
+ TableMeta meta = CatalogUtil.newTableMeta(BuiltinStorages.TEXT, util.getConfiguration());
TableDesc desc = new TableDesc(
CatalogUtil.buildFQName(TajoConstants.DEFAULT_DATABASE_NAME, "people"), schema, meta,
CommonTestingUtil.getTestDir().toUri());
diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/eval/TestPredicates.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/eval/TestPredicates.java
index cc9da7f..32403b3 100644
--- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/eval/TestPredicates.java
+++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/eval/TestPredicates.java
@@ -295,7 +295,7 @@
.build();
testEval(schema2, "table1", "a,b,c", "select col1 in ('a'), col2 in ('a', 'c') from table1", new String[]{"t","f"});
- testEval(schema2, "table1", "a,,c", "select col1 in ('a','b','c'), (col2 in ('a', 'c')) is null from table1",
+ testEval(schema2, "table1", "a,\\NULL,c", "select col1 in ('a','b','c'), (col2 in ('a', 'c')) is null from table1",
new String[]{"t","t"});
testEval(schema2,
@@ -307,7 +307,7 @@
// null handling test
testEval(schema2,
"table1",
- "2014-03-21,,2015-04-01",
+ "2014-03-21,\\NULL,2015-04-01",
"select (substr(col2,1,4)::int4 in (2014,2015,2016)) is null from table1",
new String[]{"t"});
}
@@ -355,10 +355,10 @@
.add("col1", TEXT)
.add("col2", TEXT)
.build();
- testEval(schema2, "table1", "_123,", "select ltrim(col1, '_') is null, upper(col2) is null as a from table1",
+ testEval(schema2, "table1", "_123,\\NULL", "select ltrim(col1, '_') is null, upper(col2) is null as a from table1",
new String[]{"f", "t"});
- testEval(schema2, "table1", "_123,",
+ testEval(schema2, "table1", "_123,\\NULL",
"select ltrim(col1, '_') is not null, upper(col2) is not null as a from table1", new String[]{"t", "f"});
}
diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/eval/TestSQLExpression.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/eval/TestSQLExpression.java
index 293b02a..f2fb792 100644
--- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/eval/TestSQLExpression.java
+++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/eval/TestSQLExpression.java
@@ -26,14 +26,15 @@
import org.apache.tajo.datum.DatumFactory;
import org.apache.tajo.datum.TimestampDatum;
import org.apache.tajo.engine.query.QueryContext;
+import org.apache.tajo.exception.InvalidOperationException;
import org.apache.tajo.exception.TajoException;
import org.apache.tajo.exception.UndefinedFunctionException;
-import org.apache.tajo.util.datetime.DateTimeUtil;
import org.junit.Test;
import java.util.TimeZone;
import static org.apache.tajo.common.TajoDataTypes.Type.*;
+import static org.junit.Assert.assertEquals;
import static org.junit.Assert.fail;
public class TestSQLExpression extends ExprTestBase {
@@ -817,8 +818,6 @@
@Test
public void testCastFromTable() throws TajoException {
QueryContext queryContext = new QueryContext(getConf());
- queryContext.put(SessionVars.TIMEZONE, "GMT-6");
- TimeZone tz = TimeZone.getTimeZone("GMT-6");
Schema schema = SchemaBuilder.builder()
.add("col1", TEXT)
@@ -833,16 +832,12 @@
testEval(queryContext, schema, "table1", "123,234", "select col1::float, col2::float from table1",
new String[]{"123.0", "234.0"});
- TimestampDatum timestamp = DatumFactory.createTimestamp("1980-04-01 01:50:01" +
- DateTimeUtil.getTimeZoneDisplayTime(tz));
-
testEval(queryContext, schema, "table1", "1980-04-01 01:50:01,234",
"select col1::timestamp as t1, col2::float from table1 where t1 = '1980-04-01 01:50:01'::timestamp",
- new String[]{TimestampDatum.asChars(timestamp.asTimeMeta(), tz, false), "234.0"}
+ new String[]{"1980-04-01 01:50:01", "234.0"}
);
- testSimpleEval("select '1980-04-01 01:50:01'::timestamp;", new String[]{
- TimestampDatum.asChars(timestamp.asTimeMeta(), tz, false)});
+ testSimpleEval("select '1980-04-01 01:50:01'::timestamp;", new String[]{"1980-04-01 01:50:01"});
testSimpleEval("select '1980-04-01 01:50:01'::timestamp::text", new String[]{"1980-04-01 01:50:01"});
testSimpleEval("select (cast ('99999'::int8 as text))::int4 + 1", new String[]{"100000"});
@@ -950,4 +945,81 @@
testSimpleEval("select (false OR 1 > null) is null", new String[] {"t"}); // false - unknown -> unknown
testSimpleEval("select (false OR false)", new String[] {"f"}); // false - false -> false
}
+
+ @Test
+ public void testInvalidOperation() throws TajoException {
+ testEvalException("select '1980-09-04'::date + '1980-09-04 00:10:10'::timestamp", InvalidOperationException.class);
+ testEvalException("select '1980-09-04 00:10:10'::timestamp + '1980-09-04'::date", InvalidOperationException.class);
+
+ testEvalException("select time '00:00' + time '02:00'", InvalidOperationException.class);
+ testEvalException("select time '00:00' - '1980-09-04'::date", InvalidOperationException.class);
+ testEvalException("select time '00:00' - '1980-09-04 00:10:10'::timestamp", InvalidOperationException.class);
+ testEvalException("select interval '1 day' - '1980-09-04 00:10:10'::timestamp", InvalidOperationException.class);
+
+ //TODO this operation should be allowed after timestamptz added
+ testEvalException("select date '1980-09-04' < timestamp '1980-09-04 00:00:01'", InvalidOperationException.class);
+ testEvalException("select date '1980-09-04' > timestamp '1980-09-04 00:00:01'", InvalidOperationException.class);
+ testEvalException("select date '1980-09-04' = timestamp '1980-09-04 00:00:01'", InvalidOperationException.class);
+ testEvalException("select '1980-09-04 00:10:10'::timestamp - '1980-09-04'::date", InvalidOperationException.class);
+ testEvalException("select '1980-09-04'::date - '1980-09-04 00:10:10'::timestamp", InvalidOperationException.class);
+ }
+
+ @Test
+ public void testArithmeticOperandForDateTime() throws TajoException {
+ Schema schema = SchemaBuilder.builder()
+ .add("col0", TIME)
+ .add("col1", DATE)
+ .add("col2", TIMESTAMP)
+ .build();
+
+ testEval(schema, "table1", "01:00:00,1980-09-04,1980-09-04 01:00:00",
+ "select col0 + col1 from table1;", new String[]{"1980-09-04 01:00:00"});
+ testEval(schema, "table1", "01:00:00,1980-09-04,1980-09-04 01:00:00",
+ "select col0 + col2 from table1;", new String[]{"1980-09-04 02:00:00"});
+ testEval(schema, "table1", "01:00:00,1980-09-04,1980-09-04 01:00:00",
+ "select col0 + interval '1 hour' from table1;", new String[]{"02:00:00"});
+ testEval(schema, "table1", "01:00:00,1980-09-04,1980-09-04 01:00:00",
+ "select interval '1 hour' + col0 from table1;", new String[]{"02:00:00"});
+
+ testEval(schema, "table1", "01:00:00,1980-09-04,1980-09-04 01:00:00",
+ "select col1 + interval '1 day' from table1;", new String[]{"1980-09-05 00:00:00"});
+ testEval(schema, "table1", "01:00:00,1980-09-04,1980-09-04 01:00:00",
+ "select interval '1 day' + col1 from table1;", new String[]{"1980-09-05 00:00:00"});
+ testEval(schema, "table1", "01:00:00,1980-09-04,1980-09-04 01:00:00",
+ "select col1 + col0 from table1;", new String[]{"1980-09-04 01:00:00"});
+
+ testEval(schema, "table1", "01:00:00,1980-09-04,1980-09-04 01:00:00",
+ "select col2 + (interval '1 day' + interval '1 hour') from table1;", new String[]{"1980-09-05 02:00:00"});
+ testEval(schema, "table1", "01:00:00,1980-09-04,1980-09-04 01:00:00",
+ "select (interval '1 day' + interval '1 hour') + col2 from table1;", new String[]{"1980-09-05 02:00:00"});
+ testEval(schema, "table1", "01:00:00,1980-09-04,1980-09-04 01:00:00",
+ "select col2 + col0 from table1;", new String[]{"1980-09-04 02:00:00"});
+
+ testEval(schema, "table1", "01:00:00,1980-09-04,1980-09-04 01:00:00",
+ "select col0 - col0 from table1;", new String[]{"00:00:00"});
+ testEval(schema, "table1", "01:00:00,1980-09-04,1980-09-04 01:00:00",
+ "select col1 - col1 from table1;", new String[]{"0"});
+ testEval(schema, "table1", "01:00:00,1980-09-04,1980-09-04 01:00:00",
+ "select col2 - col2 from table1;", new String[]{"00:00:00"});
+
+ testEval(schema, "table1", "01:00:00,1980-09-04,1980-09-04 01:00:00",
+ "select col0 - interval '1 hour' from table1;", new String[]{"00:00:00"});
+ testEval(schema, "table1", "01:00:00,1980-09-04,1980-09-04 01:00:00",
+ "select col1 - interval '1 day' from table1;", new String[]{"1980-09-03 00:00:00"});
+ testEval(schema, "table1", "01:00:00,1980-09-04,1980-09-04 01:00:00",
+ "select col1 - col0 from table1;", new String[]{"1980-09-03 23:00:00"});
+ testEval(schema, "table1", "01:00:00,1980-09-04,1980-09-04 01:00:00",
+ "select col2 - (interval '1 day' + interval '1 hour') from table1;", new String[]{"1980-09-03 00:00:00"});
+ testEval(schema, "table1", "01:00:00,1980-09-04,1980-09-04 01:00:00",
+ "select col2 - col0 from table1;", new String[]{"1980-09-04 00:00:00"});
+ }
+
+ private <T extends Throwable> void testEvalException(String query, Class<T> clazz) {
+ try {
+ testSimpleEval(query, new String[]{""});
+ fail(query);
+ } catch (Throwable e) {
+ assertEquals(e.getMessage(), clazz, e.getClass());
+ }
+ }
}
diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/function/TestConditionalExpressions.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/function/TestConditionalExpressions.java
index 20045ad..8d3a94a 100644
--- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/function/TestConditionalExpressions.java
+++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/function/TestConditionalExpressions.java
@@ -69,7 +69,7 @@
testEval(schema, "table1", "str1,str2",
"SELECT CASE WHEN col1 IS NOT NULL THEN col2 ELSE NULL END FROM table1",
new String[]{"str2"});
- testEval(schema, "table1", ",str2",
+ testEval(schema, "table1", "\\NULL,str2",
"SELECT CASE WHEN col1 IS NOT NULL THEN col2 ELSE NULL END FROM table1",
new String[]{NullDatum.get().toString()});
}
diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/function/TestDateTimeFunctions.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/function/TestDateTimeFunctions.java
index 3682006..bf997f3 100644
--- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/function/TestDateTimeFunctions.java
+++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/function/TestDateTimeFunctions.java
@@ -22,8 +22,6 @@
import org.apache.tajo.SessionVars;
import org.apache.tajo.catalog.Schema;
import org.apache.tajo.catalog.SchemaBuilder;
-import org.apache.tajo.datum.DatumFactory;
-import org.apache.tajo.datum.TimestampDatum;
import org.apache.tajo.engine.eval.ExprTestBase;
import org.apache.tajo.engine.query.QueryContext;
import org.apache.tajo.exception.TajoException;
@@ -31,8 +29,8 @@
import org.apache.tajo.util.datetime.TimeMeta;
import org.junit.Test;
-import java.text.SimpleDateFormat;
-import java.util.Date;
+import java.time.ZonedDateTime;
+import java.time.format.DateTimeFormatter;
import java.util.TimeZone;
import static org.apache.tajo.common.TajoDataTypes.Type.*;
@@ -40,12 +38,12 @@
public class TestDateTimeFunctions extends ExprTestBase {
@Test
public void testToTimestamp() throws TajoException {
- long expectedTimestamp = System.currentTimeMillis();
- TimestampDatum expected = DatumFactory.createTimestampDatumWithUnixTime((int)(expectedTimestamp/ 1000));
- // (expectedTimestamp / 1000) means the translation from millis seconds to unix timestamp
- String q1 = String.format("select to_timestamp(%d);", (expectedTimestamp / 1000));
- testSimpleEval(q1, new String[]{expected.toString()});
+ QueryContext context = new QueryContext(getConf());
+ context.put(SessionVars.TIMEZONE, "GMT+9");
+
+ String q1 = String.format("select to_timestamp(%d);", 1389071574);
+ testSimpleEval(context, q1, new String[]{"2014-01-07 14:12:54"});
testSimpleEval("select to_timestamp('1997-12-30 11:40:50.345', 'YYYY-MM-DD HH24:MI:SS.MS');",
new String[]{"1997-12-30 11:40:50.345"});
@@ -236,9 +234,6 @@
@Test
public void testDatePart() throws TajoException {
- TimeZone GMT = TimeZone.getTimeZone("GMT");
- TimeZone PST = TimeZone.getTimeZone("PST");
-
Schema schema2 = SchemaBuilder.builder()
.add("col1", TIMESTAMP).build();
@@ -247,11 +242,11 @@
"select date_part('year', col1), date_part('month', col1), date_part('day', col1) from table1;",
new String[]{"1970.0", "1.0", "17.0"});
testEval(schema2, "table1",
- "1970-01-17 22:09:37" + getUserTimeZoneDisplay(GMT),
+ "1970-01-17 22:09:37-00",
"select date_part('year', col1), date_part('month', col1), date_part('day', col1) from table1;",
new String[]{"1970.0", "1.0", "17.0"});
testEval(schema2, "table1",
- "1970-01-17 22:09:37" + getUserTimeZoneDisplay(PST),
+ "1970-01-17 22:09:37-04",
"select date_part('year', col1), date_part('month', col1), date_part('day', col1) from table1;",
new String[]{"1970.0", "1.0", "18.0"});
@@ -261,12 +256,12 @@
testEval(schema3, "table1", "10:09:37.5",
"select date_part('hour', col1), date_part('minute', col1), date_part('second', col1) from table1;",
new String[]{"10.0", "9.0", "37.5"});
- testEval(schema3, "table1", "10:09:37.5" + getUserTimeZoneDisplay(GMT),
+ testEval(schema3, "table1", "10:09:37.5",
"select date_part('hour', col1), date_part('minute', col1), date_part('second', col1) from table1;",
new String[]{"10.0", "9.0", "37.5"});
- testEval(schema3, "table1", "10:09:37.5" + getUserTimeZoneDisplay(PST),
+ testEval(schema3, "table1", "10:09:37.5",
"select date_part('hour', col1), date_part('minute', col1), date_part('second', col1) from table1;",
- new String[]{"18.0", "9.0", "37.5"});
+ new String[]{"10.0", "9.0", "37.5"});
Schema schema4 = SchemaBuilder.builder()
.add("col1", DATE)
@@ -431,59 +426,43 @@
@Test
public void testDateTimeNow() throws TajoException {
- TimeZone originalTimezone = TimeZone.getDefault();
- TimeZone.setDefault(TimeZone.getTimeZone("GMT-6"));
-
QueryContext context = new QueryContext(getConf());
- context.put(SessionVars.TIMEZONE, "GMT-6");
+ context.put(SessionVars.TIMEZONE, "America/Los_Angeles");
- try {
- Date expectedDate = new Date(System.currentTimeMillis());
+ ZonedDateTime zonedDateTime = ZonedDateTime.now(TimeZone.getTimeZone(context.get(SessionVars.TIMEZONE)).toZoneId());
- testSimpleEval(context, "select to_char(now(), 'yyyy-MM-dd');",
- new String[]{dateFormat(expectedDate, "yyyy-MM-dd")});
- testSimpleEval(context, "select cast(extract(year from now()) as INT4);",
- new String[]{dateFormat(expectedDate, "yyyy")});
- testSimpleEval(context, "select current_date();",
- new String[]{dateFormat(expectedDate, "yyyy-MM-dd")});
- testSimpleEval(context, "select cast(extract(hour from current_time()) as INT4);",
- new String[]{String.valueOf(Integer.parseInt(dateFormat(expectedDate, "HH")))});
+ testSimpleEval(context, "select to_char(now(), 'yyyy-MM-dd');",
+ new String[]{dateFormat(zonedDateTime, "yyyy-MM-dd")});
+ testSimpleEval(context, "select cast(extract(year from now()) as INT4);",
+ new String[]{dateFormat(zonedDateTime, "yyyy")});
+ testSimpleEval(context, "select current_date();",
+ new String[]{dateFormat(zonedDateTime, "yyyy-MM-dd")});
+ testSimpleEval(context, "select cast(extract(hour from current_time()) as INT4);",
+ new String[]{String.valueOf(Integer.parseInt(dateFormat(zonedDateTime, "HH")))});
- expectedDate.setDate(expectedDate.getDate() + 1);
-
- testSimpleEval(context, "select current_date() + 1;",
- new String[]{dateFormat(expectedDate, "yyyy-MM-dd")});
- } finally {
- TimeZone.setDefault(originalTimezone);
- }
+ zonedDateTime = zonedDateTime.plusDays(1);
+ testSimpleEval(context, "select current_date() + 1;", new String[]{dateFormat(zonedDateTime, "yyyy-MM-dd")});
}
@Test
public void testTimeValueKeyword() throws TajoException {
- TimeZone originTimeZone = TimeZone.getDefault();
- TimeZone.setDefault(TimeZone.getTimeZone("GMT-6"));
QueryContext context = new QueryContext(getConf());
context.put(SessionVars.TIMEZONE, "GMT-6");
- try {
- Date expectedDate = new Date(System.currentTimeMillis());
+ ZonedDateTime zonedDateTime = ZonedDateTime.now(TimeZone.getTimeZone(context.get(SessionVars.TIMEZONE)).toZoneId());
- testSimpleEval(context, "select to_char(current_timestamp, 'yyyy-MM-dd');",
- new String[]{dateFormat(expectedDate, "yyyy-MM-dd")});
- testSimpleEval(context, "select cast(extract(year from current_timestamp) as INT4);",
- new String[]{dateFormat(expectedDate, "yyyy")});
- testSimpleEval(context, "select current_date;",
- new String[]{dateFormat(expectedDate, "yyyy-MM-dd")});
- testSimpleEval(context, "select cast(extract(hour from current_time) as INT4);",
- new String[]{String.valueOf(Integer.parseInt(dateFormat(expectedDate, "HH")))});
- } finally {
- TimeZone.setDefault(originTimeZone);
- }
+ testSimpleEval(context, "select to_char(current_timestamp, 'yyyy-MM-dd');",
+ new String[]{dateFormat(zonedDateTime, "yyyy-MM-dd")});
+ testSimpleEval(context, "select cast(extract(year from current_timestamp) as INT4);",
+ new String[]{dateFormat(zonedDateTime, "yyyy")});
+ testSimpleEval(context, "select current_date;",
+ new String[]{dateFormat(zonedDateTime, "yyyy-MM-dd")});
+ testSimpleEval(context, "select cast(extract(hour from current_time) as INT4);",
+ new String[]{String.valueOf(Integer.parseInt(dateFormat(zonedDateTime, "HH")))});
}
- private String dateFormat(Date date, String format) {
- SimpleDateFormat df = new SimpleDateFormat(format);
- return df.format(date);
+ private String dateFormat(ZonedDateTime dateTime, String format) {
+ return dateTime.format(DateTimeFormatter.ofPattern(format));
}
}
diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/function/TestJsonFunctions.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/function/TestJsonFunctions.java
index fc5e8e1..a7d8674 100644
--- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/function/TestJsonFunctions.java
+++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/function/TestJsonFunctions.java
@@ -51,7 +51,7 @@
public void testJsonArrayContains() throws TajoException {
testSimpleEval("select json_array_contains('" + JSON_COMPLEX_ARRAY + "', 100)", new String[]{"t"});
testSimpleEval("select json_array_contains('" + JSON_COMPLEX_ARRAY + "', 'test')", new String[]{"t"});
- testSimpleEval("select json_array_contains('" + JSON_COMPLEX_ARRAY + "', '2015-08-13 11:58:59'::timestamp)",
+ testSimpleEval("select json_array_contains('" + JSON_COMPLEX_ARRAY + "', '2015-08-13 11:58:59'::timestamp::text)",
new String[]{"t"});
testSimpleEval("select json_array_contains('" + JSON_COMPLEX_ARRAY + "', '2015-08-13 11:58:59'::date)",
new String[]{"f"});
diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/function/TestPatternMatchingPredicates.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/function/TestPatternMatchingPredicates.java
index 54907fc..f55919e 100644
--- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/function/TestPatternMatchingPredicates.java
+++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/function/TestPatternMatchingPredicates.java
@@ -36,7 +36,7 @@
.build();
// test for null values
- testEval(schema, "table1", ",", "select col1 like 'a%' from table1", new String[]{NullDatum.get().toString()});
+ testEval(schema, "table1", "\\NULL,", "select col1 like 'a%' from table1", new String[]{NullDatum.get().toString()});
testSimpleEval("select null like 'a%'", new String[]{NullDatum.get().toString()});
testEval(schema, "table1", "abc", "select col1 like '%c' from table1", new String[]{"t"});
diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/function/TestStringOperatorsAndFunctions.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/function/TestStringOperatorsAndFunctions.java
index 203af8f..a38eb71 100644
--- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/function/TestStringOperatorsAndFunctions.java
+++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/function/TestStringOperatorsAndFunctions.java
@@ -147,9 +147,9 @@
new String[]{"ab--ab"});
// null test from a table
- testEval(schema, "table1", ",(^--|--$),ab", "select regexp_replace(col1, col2, col3) as str from table1",
+ testEval(schema, "table1", "\\NULL,(^--|--$),ab", "select regexp_replace(col1, col2, col3) as str from table1",
new String[]{NullDatum.get().toString()});
- testEval(schema, "table1", "------,(^--|--$),", "select regexp_replace(col1, col2, col3) as str from table1",
+ testEval(schema, "table1", "------,(^--|--$),\\NULL", "select regexp_replace(col1, col2, col3) as str from table1",
new String[]{NullDatum.get().toString()});
}
@@ -177,7 +177,7 @@
.build();
// for null tests
- testEval(schema, "table1", ",1,ghi", "select left(col1,1) is null from table1", new String[]{"t"});
+ testEval(schema, "table1", "\\NULL,1,ghi", "select left(col1,1) is null from table1", new String[]{"t"});
testEval(schema, "table1", "abc,,ghi", "select left(col1,col2) is null from table1", new String[]{"t"});
testEval(schema, "table1", "abc,1,ghi", "select left(col1,1) || left(col3,3) from table1", new String[]{"aghi"});
@@ -207,8 +207,8 @@
.build();
// for null tests
- testEval(schema, "table1", ",1,ghi", "select right(col1,1) is null from table1", new String[]{"t"});
- testEval(schema, "table1", "abc,,ghi", "select right(col1,col2) is null from table1", new String[]{"t"});
+ testEval(schema, "table1", "\\NULL,1,ghi", "select right(col1,1) is null from table1", new String[]{"t"});
+ testEval(schema, "table1", "abc,\\NULL,ghi", "select right(col1,col2) is null from table1", new String[]{"t"});
testEval(schema, "table1", "abc,1,ghi", "select right(col1,1) || right(col3,3) from table1", new String[]{"cghi"});
}
@@ -387,7 +387,7 @@
testEval(schema, "t1", ",.,1", "select split_part(col1, col2, col3::int) is null from t1", new String[]{"t"});
testEval(schema, "t1", "1386577650.123,,1", "select split_part(col1, col2, col3::int) from t1",
new String[]{"1386577650.123"});
- testEval(schema, "t1", "1386577650.123,.,", "select split_part(col1, col2, col3::int) is null from t1",
+ testEval(schema, "t1", "1386577650.123,.,\\NULL", "select split_part(col1, col2, col3::int) is null from t1",
new String[]{"t"});
}
@@ -463,9 +463,9 @@
testEval(schema, "table1", ",abcdef,3.14", "select locate(col2, 'cd', 4) from table1", new String[]{"0"});
testEval(schema, "table1", ",abcdef,3.14", "select locate(col2, 'xy') from table1", new String[]{"0"});
// null string
- testEval(schema, "table1", ",abcdef,3.14", "select locate(col1, 'cd') is null from table1", new String[]{"t"});
+ testEval(schema, "table1", "\\NULL,abcdef,3.14", "select locate(col1, 'cd') is null from table1", new String[]{"t"});
// nul substring
- testEval(schema, "table1", ",abcdef,3.14", "select locate('cd', col1) is null from table1", new String[]{"t"});
+ testEval(schema, "table1", "\\NULL,abcdef,3.14", "select locate('cd', col1) is null from table1", new String[]{"t"});
}
@Test
@@ -617,9 +617,9 @@
.add("col1", TEXT)
.add("col2", TEXT)
.build();
- testEval(schema, "table1", "|crt,c,cr,c,def", "select find_in_set(col1, col2) is null from table1",
+ testEval(schema, "table1", "\\NULL|crt,c,cr,c,def", "select find_in_set(col1, col2) is null from table1",
new String[]{"t"}, '|', true);
- testEval(schema, "table1", "cr|", "select find_in_set(col1, col2) is null from table1",
+ testEval(schema, "table1", "cr|\\NULL", "select find_in_set(col1, col2) is null from table1",
new String[]{"t"}, '|', true);
}
diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestEvalNodeToExprConverter.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestEvalNodeToExprConverter.java
index 1cf4cf2..454ae6c 100644
--- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestEvalNodeToExprConverter.java
+++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestEvalNodeToExprConverter.java
@@ -18,6 +18,7 @@
package org.apache.tajo.engine.planner;
+import org.apache.tajo.BuiltinStorages;
import org.apache.tajo.LocalTajoTestingUtility;
import org.apache.tajo.QueryVars;
import org.apache.tajo.TajoTestingCluster;
@@ -32,7 +33,6 @@
import org.apache.tajo.plan.LogicalPlan;
import org.apache.tajo.plan.LogicalPlanner;
import org.apache.tajo.plan.expr.AlgebraicUtil;
-import org.apache.tajo.plan.expr.EvalNode;
import org.apache.tajo.plan.logical.LogicalNode;
import org.apache.tajo.plan.logical.NodeType;
import org.apache.tajo.plan.logical.ScanNode;
@@ -79,7 +79,7 @@
tpch.loadSchemas();
tpch.loadOutSchema();
for (String table : tpchTables) {
- TableMeta m = CatalogUtil.newTableMeta("TEXT");
+ TableMeta m = CatalogUtil.newTableMeta(BuiltinStorages.TEXT, util.getConfiguration());
TableDesc d = CatalogUtil.newTableDesc(
CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, table), tpch.getSchema(table), m,
CommonTestingUtil.getTestDir());
diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestJoinOrderAlgorithm.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestJoinOrderAlgorithm.java
index bd9c5f9..0805151 100644
--- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestJoinOrderAlgorithm.java
+++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestJoinOrderAlgorithm.java
@@ -20,6 +20,7 @@
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.tajo.BuiltinStorages;
import org.apache.tajo.LocalTajoTestingUtility;
import org.apache.tajo.TajoConstants;
import org.apache.tajo.TajoTestingCluster;
@@ -88,7 +89,7 @@
.add("phone", Type.INT4)
.build();
- TableMeta meta = CatalogUtil.newTableMeta("TEXT");
+ TableMeta meta = CatalogUtil.newTableMeta(BuiltinStorages.TEXT, util.getConfiguration());
TableDesc people = new TableDesc(
CatalogUtil.buildFQName(TajoConstants.DEFAULT_DATABASE_NAME, "employee"), schema, meta,
CommonTestingUtil.getTestDir().toUri());
@@ -113,7 +114,7 @@
.add("manager", Type.TEXT)
.build();
// Set store type as FAKEFILE to prevent auto update of physical information in LogicalPlanner.updatePhysicalInfo()
- TableMeta largeTableMeta = CatalogUtil.newTableMeta("FAKEFILE");
+ TableMeta largeTableMeta = CatalogUtil.newTableMeta("FAKEFILE", util.getConfiguration());
TableDesc largeDept;
TableStats largeTableStats;
FileSystem fs = FileSystem.getLocal(util.getConfiguration());
diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestLogicalOptimizer.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestLogicalOptimizer.java
index e57921d..1ff7c87 100644
--- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestLogicalOptimizer.java
+++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestLogicalOptimizer.java
@@ -18,6 +18,7 @@
package org.apache.tajo.engine.planner;
+import org.apache.tajo.BuiltinStorages;
import org.apache.tajo.LocalTajoTestingUtility;
import org.apache.tajo.TajoConstants;
import org.apache.tajo.TajoTestingCluster;
@@ -82,7 +83,7 @@
.add("phone", Type.INT4)
.build();
- TableMeta meta = CatalogUtil.newTableMeta("TEXT");
+ TableMeta meta = CatalogUtil.newTableMeta(BuiltinStorages.TEXT, util.getConfiguration());
TableDesc people = new TableDesc(
CatalogUtil.buildFQName(TajoConstants.DEFAULT_DATABASE_NAME, "employee"), schema, meta,
CommonTestingUtil.getTestDir().toUri());
diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java
index d01b0d5..c50a21d 100644
--- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java
+++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java
@@ -21,10 +21,7 @@
import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
-import org.apache.tajo.LocalTajoTestingUtility;
-import org.apache.tajo.QueryVars;
-import org.apache.tajo.TajoConstants;
-import org.apache.tajo.TajoTestingCluster;
+import org.apache.tajo.*;
import org.apache.tajo.algebra.AlterTableOpType;
import org.apache.tajo.algebra.Expr;
import org.apache.tajo.algebra.JoinType;
@@ -102,7 +99,7 @@
.add("score", Type.INT4)
.build();
- TableMeta meta = CatalogUtil.newTableMeta("TEXT");
+ TableMeta meta = CatalogUtil.newTableMeta(BuiltinStorages.TEXT, util.getConfiguration());
TableDesc people = new TableDesc(
CatalogUtil.buildFQName(TajoConstants.DEFAULT_DATABASE_NAME, "employee"), schema, meta,
CommonTestingUtil.getTestDir().toUri());
@@ -131,7 +128,7 @@
tpch.loadSchemas();
tpch.loadOutSchema();
for (String table : tpchTables) {
- TableMeta m = CatalogUtil.newTableMeta("TEXT");
+ TableMeta m = CatalogUtil.newTableMeta(BuiltinStorages.TEXT, util.getConfiguration());
TableDesc d = CatalogUtil.newTableDesc(
CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, table), tpch.getSchema(table), m,
CommonTestingUtil.getTestDir());
@@ -1335,7 +1332,8 @@
@Test
public void testSelectFromSelfDescTable() throws Exception {
- TableDesc tableDesc = new TableDesc("default.self_desc_table1", null, CatalogUtil.newTableMeta("TEXT"),
+ TableDesc tableDesc = new TableDesc("default.self_desc_table1", null,
+ CatalogUtil.newTableMeta(BuiltinStorages.TEXT, util.getConfiguration()),
CommonTestingUtil.getTestDir().toUri(), true);
catalog.createTable(tableDesc);
assertTrue(catalog.existsTable("default.self_desc_table1"));
@@ -1388,7 +1386,8 @@
@Test
public void testSelectWhereFromSelfDescTable() throws Exception {
- TableDesc tableDesc = new TableDesc("default.self_desc_table1", null, CatalogUtil.newTableMeta("TEXT"),
+ TableDesc tableDesc = new TableDesc("default.self_desc_table1", null,
+ CatalogUtil.newTableMeta(BuiltinStorages.TEXT, util.getConfiguration()),
CommonTestingUtil.getTestDir().toUri(), true);
catalog.createTable(tableDesc);
assertTrue(catalog.existsTable("default.self_desc_table1"));
diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestPlannerUtil.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestPlannerUtil.java
index 92557f1..7fd249d 100644
--- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestPlannerUtil.java
+++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestPlannerUtil.java
@@ -22,6 +22,7 @@
import org.apache.hadoop.fs.LocatedFileStatus;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.RemoteIterator;
+import org.apache.tajo.BuiltinStorages;
import org.apache.tajo.LocalTajoTestingUtility;
import org.apache.tajo.TajoConstants;
import org.apache.tajo.TajoTestingCluster;
@@ -85,7 +86,7 @@
.add("deptname", Type.TEXT)
.add("score", CatalogUtil.newSimpleDataType(Type.INT4)).build();
- TableMeta meta = CatalogUtil.newTableMeta("TEXT");
+ TableMeta meta = CatalogUtil.newTableMeta(BuiltinStorages.TEXT, util.getConfiguration());
TableDesc people = new TableDesc(
CatalogUtil.buildFQName(TajoConstants.DEFAULT_DATABASE_NAME, "employee"), schema, meta,
CommonTestingUtil.getTestDir().toUri());
diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestExternalSortExec.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestExternalSortExec.java
index 5481d07..cb49552 100644
--- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestExternalSortExec.java
+++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestExternalSortExec.java
@@ -19,10 +19,7 @@
package org.apache.tajo.engine.planner.physical;
import org.apache.hadoop.fs.Path;
-import org.apache.tajo.LocalTajoTestingUtility;
-import org.apache.tajo.SessionVars;
-import org.apache.tajo.TajoConstants;
-import org.apache.tajo.TajoTestingCluster;
+import org.apache.tajo.*;
import org.apache.tajo.algebra.Expr;
import org.apache.tajo.catalog.*;
import org.apache.tajo.common.TajoDataTypes.Type;
@@ -121,7 +118,7 @@
new Column("col12", Type.INT8)
}).build();
- TableMeta employeeMeta = CatalogUtil.newTableMeta("TEXT");
+ TableMeta employeeMeta = CatalogUtil.newTableMeta(BuiltinStorages.TEXT, util.getConfiguration());
Path employeePath = new Path(testDir, "employee.csv");
Appender appender = ((FileTablespace) TablespaceManager.getLocalFs())
.getAppender(employeeMeta, tableSchema, employeePath);
diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterHashJoinExec.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterHashJoinExec.java
index 1d2e6b5..5c56aef 100644
--- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterHashJoinExec.java
+++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterHashJoinExec.java
@@ -19,6 +19,7 @@
package org.apache.tajo.engine.planner.physical;
import org.apache.hadoop.fs.Path;
+import org.apache.tajo.BuiltinStorages;
import org.apache.tajo.LocalTajoTestingUtility;
import org.apache.tajo.TajoTestingCluster;
import org.apache.tajo.algebra.Expr;
@@ -108,7 +109,7 @@
.build();
- TableMeta dep3Meta = CatalogUtil.newTableMeta("TEXT");
+ TableMeta dep3Meta = CatalogUtil.newTableMeta(BuiltinStorages.TEXT, util.getConfiguration());
Path dep3Path = new Path(testDir, "dep3.csv");
Appender appender1 = ((FileTablespace) TablespaceManager.getLocalFs()).getAppender(dep3Meta, dep3Schema, dep3Path);
appender1.init();
@@ -138,7 +139,7 @@
.build();
- TableMeta job3Meta = CatalogUtil.newTableMeta("TEXT");
+ TableMeta job3Meta = CatalogUtil.newTableMeta(BuiltinStorages.TEXT, util.getConfiguration());
Path job3Path = new Path(testDir, "job3.csv");
Appender appender2 = ((FileTablespace) TablespaceManager.getLocalFs()).getAppender(job3Meta, job3Schema, job3Path);
appender2.init();
@@ -178,7 +179,7 @@
.build();
- TableMeta emp3Meta = CatalogUtil.newTableMeta("TEXT");
+ TableMeta emp3Meta = CatalogUtil.newTableMeta(BuiltinStorages.TEXT, util.getConfiguration());
Path emp3Path = new Path(testDir, "emp3.csv");
Appender appender3 = ((FileTablespace) TablespaceManager.getLocalFs()).getAppender(emp3Meta, emp3Schema, emp3Path);
appender3.init();
@@ -231,7 +232,7 @@
.build();
- TableMeta phone3Meta = CatalogUtil.newTableMeta("TEXT");
+ TableMeta phone3Meta = CatalogUtil.newTableMeta(BuiltinStorages.TEXT, util.getConfiguration());
Path phone3Path = new Path(testDir, "phone3.csv");
Appender appender5 = ((FileTablespace) TablespaceManager.getLocalFs())
.getAppender(phone3Meta, phone3Schema, phone3Path);
diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterMergeJoinExec.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterMergeJoinExec.java
index 0fb7bbb..605ad6e 100644
--- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterMergeJoinExec.java
+++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterMergeJoinExec.java
@@ -19,6 +19,7 @@
package org.apache.tajo.engine.planner.physical;
import org.apache.hadoop.fs.Path;
+import org.apache.tajo.BuiltinStorages;
import org.apache.tajo.LocalTajoTestingUtility;
import org.apache.tajo.TajoConstants;
import org.apache.tajo.TajoTestingCluster;
@@ -112,7 +113,7 @@
.build();
- TableMeta dep3Meta = CatalogUtil.newTableMeta("TEXT");
+ TableMeta dep3Meta = CatalogUtil.newTableMeta(BuiltinStorages.TEXT, util.getConfiguration());
Path dep3Path = new Path(testDir, "dep3.csv");
Appender appender1 = ((FileTablespace) TablespaceManager.getLocalFs()).getAppender(dep3Meta, dep3Schema, dep3Path);
appender1.init();
@@ -151,7 +152,7 @@
.build();
- TableMeta dep4Meta = CatalogUtil.newTableMeta("TEXT");
+ TableMeta dep4Meta = CatalogUtil.newTableMeta(BuiltinStorages.TEXT, util.getConfiguration());
Path dep4Path = new Path(testDir, "dep4.csv");
Appender appender4 = ((FileTablespace) TablespaceManager.getLocalFs()).getAppender(dep4Meta, dep4Schema, dep4Path);
appender4.init();
@@ -183,7 +184,7 @@
.build();
- TableMeta job3Meta = CatalogUtil.newTableMeta("TEXT");
+ TableMeta job3Meta = CatalogUtil.newTableMeta(BuiltinStorages.TEXT, util.getConfiguration());
Path job3Path = new Path(testDir, "job3.csv");
Appender appender2 = ((FileTablespace) TablespaceManager.getLocalFs()).getAppender(job3Meta, job3Schema, job3Path);
appender2.init();
@@ -223,7 +224,7 @@
.build();
- TableMeta emp3Meta = CatalogUtil.newTableMeta("TEXT");
+ TableMeta emp3Meta = CatalogUtil.newTableMeta(BuiltinStorages.TEXT, util.getConfiguration());
Path emp3Path = new Path(testDir, "emp3.csv");
Appender appender3 = ((FileTablespace) TablespaceManager.getLocalFs()).getAppender(emp3Meta, emp3Schema, emp3Path);
appender3.init();
@@ -276,7 +277,7 @@
.build();
- TableMeta phone3Meta = CatalogUtil.newTableMeta("TEXT");
+ TableMeta phone3Meta = CatalogUtil.newTableMeta(BuiltinStorages.TEXT, util.getConfiguration());
Path phone3Path = new Path(testDir, "phone3.csv");
Appender appender5 = ((FileTablespace) TablespaceManager.getLocalFs())
.getAppender(phone3Meta, phone3Schema, phone3Path);
diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestHashAntiJoinExec.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestHashAntiJoinExec.java
index cd04ea0..1398c4d 100644
--- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestHashAntiJoinExec.java
+++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestHashAntiJoinExec.java
@@ -19,6 +19,7 @@
package org.apache.tajo.engine.planner.physical;
import org.apache.hadoop.fs.Path;
+import org.apache.tajo.BuiltinStorages;
import org.apache.tajo.LocalTajoTestingUtility;
import org.apache.tajo.TajoTestingCluster;
import org.apache.tajo.algebra.Expr;
@@ -85,7 +86,7 @@
.add("deptname", Type.TEXT)
.build();
- TableMeta employeeMeta = CatalogUtil.newTableMeta("TEXT");
+ TableMeta employeeMeta = CatalogUtil.newTableMeta(BuiltinStorages.TEXT, util.getConfiguration());
Path employeePath = new Path(testDir, "employee.csv");
Appender appender = ((FileTablespace) TablespaceManager.getLocalFs())
.getAppender(employeeMeta, employeeSchema, employeePath);
@@ -112,7 +113,7 @@
.add("name", Type.TEXT)
.add("age", Type.INT4)
.build();
- TableMeta peopleMeta = CatalogUtil.newTableMeta("TEXT");
+ TableMeta peopleMeta = CatalogUtil.newTableMeta(BuiltinStorages.TEXT, util.getConfiguration());
Path peoplePath = new Path(testDir, "people.csv");
appender = ((FileTablespace) TablespaceManager.getLocalFs())
.getAppender(peopleMeta, peopleSchema, peoplePath);
diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestHashJoinExec.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestHashJoinExec.java
index cdf9068..191c346 100644
--- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestHashJoinExec.java
+++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestHashJoinExec.java
@@ -19,10 +19,7 @@
package org.apache.tajo.engine.planner.physical;
import org.apache.hadoop.fs.Path;
-import org.apache.tajo.LocalTajoTestingUtility;
-import org.apache.tajo.SessionVars;
-import org.apache.tajo.TajoConstants;
-import org.apache.tajo.TajoTestingCluster;
+import org.apache.tajo.*;
import org.apache.tajo.algebra.Expr;
import org.apache.tajo.catalog.*;
import org.apache.tajo.common.TajoDataTypes.Type;
@@ -87,7 +84,7 @@
.add("deptname", Type.TEXT)
.build();
- TableMeta employeeMeta = CatalogUtil.newTableMeta("TEXT");
+ TableMeta employeeMeta = CatalogUtil.newTableMeta(BuiltinStorages.TEXT, util.getConfiguration());
Path employeePath = new Path(testDir, "employee.csv");
Appender appender = ((FileTablespace) TablespaceManager.getLocalFs())
.getAppender(employeeMeta, employeeSchema, employeePath);
@@ -111,7 +108,8 @@
.add("name", Type.TEXT)
.add("age", Type.INT4)
.build();
- TableMeta peopleMeta = CatalogUtil.newTableMeta("TEXT");
+
+ TableMeta peopleMeta = CatalogUtil.newTableMeta(BuiltinStorages.TEXT, util.getConfiguration());
Path peoplePath = new Path(testDir, "people.csv");
appender = ((FileTablespace) TablespaceManager.getLocalFs())
.getAppender(peopleMeta, peopleSchema, peoplePath);
diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestHashSemiJoinExec.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestHashSemiJoinExec.java
index 026bbe2..baebf80 100644
--- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestHashSemiJoinExec.java
+++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestHashSemiJoinExec.java
@@ -19,6 +19,7 @@
package org.apache.tajo.engine.planner.physical;
import org.apache.hadoop.fs.Path;
+import org.apache.tajo.BuiltinStorages;
import org.apache.tajo.LocalTajoTestingUtility;
import org.apache.tajo.TajoTestingCluster;
import org.apache.tajo.algebra.Expr;
@@ -85,7 +86,7 @@
.add("deptname", Type.TEXT)
.build();
- TableMeta employeeMeta = CatalogUtil.newTableMeta("TEXT");
+ TableMeta employeeMeta = CatalogUtil.newTableMeta(BuiltinStorages.TEXT, util.getConfiguration());
Path employeePath = new Path(testDir, "employee.csv");
Appender appender = ((FileTablespace) TablespaceManager.getLocalFs())
.getAppender(employeeMeta, employeeSchema, employeePath);
@@ -112,7 +113,8 @@
.add("name", Type.TEXT)
.add("age", Type.INT4)
.build();
- TableMeta peopleMeta = CatalogUtil.newTableMeta("TEXT");
+
+ TableMeta peopleMeta = CatalogUtil.newTableMeta(BuiltinStorages.TEXT, util.getConfiguration());
Path peoplePath = new Path(testDir, "people.csv");
appender = ((FileTablespace) TablespaceManager.getLocalFs())
.getAppender(peopleMeta, peopleSchema, peoplePath);
diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterHashJoinExec.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterHashJoinExec.java
index c308065..fa20941 100644
--- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterHashJoinExec.java
+++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterHashJoinExec.java
@@ -19,6 +19,7 @@
package org.apache.tajo.engine.planner.physical;
import org.apache.hadoop.fs.Path;
+import org.apache.tajo.BuiltinStorages;
import org.apache.tajo.LocalTajoTestingUtility;
import org.apache.tajo.TajoTestingCluster;
import org.apache.tajo.algebra.Expr;
@@ -106,7 +107,7 @@
.build();
- TableMeta dep3Meta = CatalogUtil.newTableMeta("TEXT");
+ TableMeta dep3Meta = CatalogUtil.newTableMeta(BuiltinStorages.TEXT, util.getConfiguration());
Path dep3Path = new Path(testDir, "dep3.csv");
Appender appender1 = ((FileTablespace) TablespaceManager.getLocalFs()).getAppender(dep3Meta, dep3Schema, dep3Path);
appender1.init();
@@ -136,7 +137,7 @@
.build();
- TableMeta job3Meta = CatalogUtil.newTableMeta("TEXT");
+ TableMeta job3Meta = CatalogUtil.newTableMeta(BuiltinStorages.TEXT, util.getConfiguration());
Path job3Path = new Path(testDir, "job3.csv");
Appender appender2 = ((FileTablespace) TablespaceManager.getLocalFs())
.getAppender(job3Meta, job3Schema, job3Path);
@@ -177,7 +178,7 @@
.build();
- TableMeta emp3Meta = CatalogUtil.newTableMeta("TEXT");
+ TableMeta emp3Meta = CatalogUtil.newTableMeta(BuiltinStorages.TEXT, util.getConfiguration());
Path emp3Path = new Path(testDir, "emp3.csv");
Appender appender3 = ((FileTablespace) TablespaceManager.getLocalFs())
.getAppender(emp3Meta, emp3Schema, emp3Path);
@@ -231,7 +232,7 @@
.build();
- TableMeta phone3Meta = CatalogUtil.newTableMeta("TEXT");
+ TableMeta phone3Meta = CatalogUtil.newTableMeta(BuiltinStorages.TEXT, util.getConfiguration());
Path phone3Path = new Path(testDir, "phone3.csv");
Appender appender5 = ((FileTablespace) TablespaceManager.getLocalFs())
.getAppender(phone3Meta, phone3Schema, phone3Path);
diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestMergeJoinExec.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestMergeJoinExec.java
index 5819003..d3d1c24 100644
--- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestMergeJoinExec.java
+++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestMergeJoinExec.java
@@ -19,6 +19,7 @@
package org.apache.tajo.engine.planner.physical;
import org.apache.hadoop.fs.Path;
+import org.apache.tajo.BuiltinStorages;
import org.apache.tajo.LocalTajoTestingUtility;
import org.apache.tajo.TajoConstants;
import org.apache.tajo.TajoTestingCluster;
@@ -85,7 +86,7 @@
.add("deptname", Type.TEXT)
.build();
- TableMeta employeeMeta = CatalogUtil.newTableMeta("TEXT");
+ TableMeta employeeMeta = CatalogUtil.newTableMeta(BuiltinStorages.TEXT, util.getConfiguration());
Path employeePath = new Path(testDir, "employee.csv");
Appender appender = ((FileTablespace) TablespaceManager.getLocalFs())
.getAppender(employeeMeta, employeeSchema, employeePath);
@@ -115,7 +116,8 @@
.add("name", Type.TEXT)
.add("age", Type.INT4)
.build();
- TableMeta peopleMeta = CatalogUtil.newTableMeta("TEXT");
+
+ TableMeta peopleMeta = CatalogUtil.newTableMeta(BuiltinStorages.TEXT, util.getConfiguration());
Path peoplePath = new Path(testDir, "people.csv");
appender = ((FileTablespace) TablespaceManager.getLocalFs())
.getAppender(peopleMeta, peopleSchema, peoplePath);
diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java
index 694193d..c3a669e 100644
--- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java
+++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java
@@ -117,7 +117,7 @@
.add("nullable", Type.TEXT)
.build();
- TableMeta employeeMeta = CatalogUtil.newTableMeta("TEXT");
+ TableMeta employeeMeta = CatalogUtil.newTableMeta(BuiltinStorages.TEXT, conf);
Path employeePath = new Path(testDir, "employee.csv");
@@ -138,7 +138,7 @@
catalog.createTable(employee);
Path scorePath = new Path(testDir, "score");
- TableMeta scoreMeta = CatalogUtil.newTableMeta("TEXT", new KeyValueSet());
+ TableMeta scoreMeta = CatalogUtil.newTableMeta(BuiltinStorages.TEXT, conf);
appender = sm.getAppender(scoreMeta, scoreSchema, scorePath);
appender.init();
score = new TableDesc(
@@ -435,7 +435,7 @@
LogicalPlan plan = planner.createPlan(defaultContext, context);
LogicalNode rootNode = optimizer.optimize(plan);
- TableMeta outputMeta = CatalogUtil.newTableMeta("TEXT");
+ TableMeta outputMeta = CatalogUtil.newTableMeta(BuiltinStorages.TEXT, conf);;
PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf);
PhysicalExec exec = phyPlanner.createPlan(ctx, rootNode);
@@ -504,7 +504,7 @@
long totalNum = 0;
for (FileStatus status : fs.listStatus(ctx.getOutputPath().getParent())) {
Scanner scanner = ((FileTablespace) TablespaceManager.getLocalFs()).getFileScanner(
- CatalogUtil.newTableMeta("TEXT"),
+ CatalogUtil.newTableMeta(BuiltinStorages.TEXT, conf),
rootNode.getOutSchema(),
status.getPath());
@@ -532,7 +532,7 @@
LogicalPlan plan = planner.createPlan(defaultContext, context);
LogicalNode rootNode = optimizer.optimize(plan);
- TableMeta outputMeta = CatalogUtil.newTableMeta("RCFILE");
+ TableMeta outputMeta = CatalogUtil.newTableMeta(BuiltinStorages.RCFILE, conf);
PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf);
PhysicalExec exec = phyPlanner.createPlan(ctx, rootNode);
@@ -648,7 +648,7 @@
ctx.setDataChannel(dataChannel);
LogicalNode rootNode = optimizer.optimize(plan);
- TableMeta outputMeta = CatalogUtil.newTableMeta(dataChannel.getDataFormat());
+ TableMeta outputMeta = CatalogUtil.newTableMeta(dataChannel.getDataFormat(), conf);
FileSystem fs = sm.getFileSystem();
QueryId queryId = id.getTaskId().getExecutionBlockId().getQueryId();
@@ -745,7 +745,7 @@
long expectedFileNum = (long) Math.ceil(fileVolumSum / (float)StorageUnit.MB);
assertEquals(expectedFileNum, fileStatuses.length);
}
- TableMeta outputMeta = CatalogUtil.newTableMeta("TEXT");
+ TableMeta outputMeta = CatalogUtil.newTableMeta(BuiltinStorages.TEXT, conf);;
Scanner scanner = new MergeScanner(conf, rootNode.getOutSchema(), outputMeta, new ArrayList<>(fragments));
scanner.init();
@@ -782,7 +782,7 @@
ctx.setDataChannel(dataChannel);
optimizer.optimize(plan);
- TableMeta outputMeta = CatalogUtil.newTableMeta(dataChannel.getDataFormat());
+ TableMeta outputMeta = CatalogUtil.newTableMeta(dataChannel.getDataFormat(), conf);
FileSystem fs = sm.getFileSystem();
QueryId queryId = id.getTaskId().getExecutionBlockId().getQueryId();
diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestProgressExternalSortExec.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestProgressExternalSortExec.java
index 51cd5ea..f88e2ef 100644
--- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestProgressExternalSortExec.java
+++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestProgressExternalSortExec.java
@@ -86,7 +86,7 @@
.add("deptname", TajoDataTypes.Type.TEXT)
.build();
- TableMeta employeeMeta = CatalogUtil.newTableMeta(BuiltinStorages.RAW);
+ TableMeta employeeMeta = CatalogUtil.newTableMeta(BuiltinStorages.RAW, conf);
Path employeePath = new Path(testDir, "employee.raw");
Appender appender = ((FileTablespace) TablespaceManager.getLocalFs())
.getAppender(employeeMeta, schema, employeePath);
diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestRadixSort.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestRadixSort.java
index f357379..ed5c3f8 100644
--- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestRadixSort.java
+++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestRadixSort.java
@@ -36,6 +36,8 @@
import org.apache.tajo.tuple.memory.UnSafeTuple;
import org.apache.tajo.tuple.memory.UnSafeTupleList;
import org.apache.tajo.util.StringUtils;
+import org.apache.tajo.util.datetime.DateTimeConstants;
+import org.apache.tajo.util.datetime.DateTimeUtil;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
@@ -219,7 +221,10 @@
DatumFactory.createInt4(Integer.MAX_VALUE),
DatumFactory.createInt2(Short.MAX_VALUE),
DatumFactory.createDate(Integer.MAX_VALUE),
- DatumFactory.createTimestamp(Long.MAX_VALUE),
+ DatumFactory.createTimestamp(
+ // FIXME 'Out of Range of Time'
+ //DateTimeUtil.toJulianDate(JULIAN_MAXYEAR, 1, 1)
+ DateTimeUtil.toJulianTimestamp(DateTimeConstants.JULIAN_MAXYEAR / 20, 1, 1, 0, 0, 0, 0)),
DatumFactory.createTime(Long.MAX_VALUE),
DatumFactory.createInet4(Integer.MAX_VALUE),
DatumFactory.createFloat4(Float.MAX_VALUE),
diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterHashJoinExec.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterHashJoinExec.java
index 83d26c7..015cf73 100644
--- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterHashJoinExec.java
+++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterHashJoinExec.java
@@ -19,6 +19,7 @@
package org.apache.tajo.engine.planner.physical;
import org.apache.hadoop.fs.Path;
+import org.apache.tajo.BuiltinStorages;
import org.apache.tajo.LocalTajoTestingUtility;
import org.apache.tajo.TajoTestingCluster;
import org.apache.tajo.algebra.Expr;
@@ -100,7 +101,7 @@
.build();
- TableMeta dep3Meta = CatalogUtil.newTableMeta("TEXT");
+ TableMeta dep3Meta = CatalogUtil.newTableMeta(BuiltinStorages.TEXT, util.getConfiguration());
Path dep3Path = new Path(testDir, "dep3.csv");
Appender appender1 = ((FileTablespace) TablespaceManager.getLocalFs())
.getAppender(dep3Meta, dep3Schema, dep3Path);
@@ -131,7 +132,7 @@
.build();
- TableMeta job3Meta = CatalogUtil.newTableMeta("TEXT");
+ TableMeta job3Meta = CatalogUtil.newTableMeta(BuiltinStorages.TEXT, util.getConfiguration());
Path job3Path = new Path(testDir, "job3.csv");
Appender appender2 = ((FileTablespace) TablespaceManager.getLocalFs())
.getAppender(job3Meta, job3Schema, job3Path);
@@ -172,7 +173,7 @@
.build();
- TableMeta emp3Meta = CatalogUtil.newTableMeta("TEXT");
+ TableMeta emp3Meta = CatalogUtil.newTableMeta(BuiltinStorages.TEXT, util.getConfiguration());
Path emp3Path = new Path(testDir, "emp3.csv");
Appender appender3 = ((FileTablespace) TablespaceManager.getLocalFs())
.getAppender(emp3Meta, emp3Schema, emp3Path);
diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterMergeJoinExec.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterMergeJoinExec.java
index 4bfe0f7..a6b5d4f 100644
--- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterMergeJoinExec.java
+++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterMergeJoinExec.java
@@ -19,6 +19,7 @@
package org.apache.tajo.engine.planner.physical;
import org.apache.hadoop.fs.Path;
+import org.apache.tajo.BuiltinStorages;
import org.apache.tajo.LocalTajoTestingUtility;
import org.apache.tajo.TajoTestingCluster;
import org.apache.tajo.algebra.Expr;
@@ -111,7 +112,7 @@
.build();
- TableMeta dep3Meta = CatalogUtil.newTableMeta("TEXT");
+ TableMeta dep3Meta = CatalogUtil.newTableMeta(BuiltinStorages.TEXT, util.getConfiguration());
Path dep3Path = new Path(testDir, "dep3.csv");
Appender appender1 = ((FileTablespace) TablespaceManager.getLocalFs())
.getAppender(dep3Meta, dep3Schema, dep3Path);
@@ -151,7 +152,7 @@
.build();
- TableMeta dep4Meta = CatalogUtil.newTableMeta("TEXT");
+ TableMeta dep4Meta = CatalogUtil.newTableMeta(BuiltinStorages.TEXT, util.getConfiguration());
Path dep4Path = new Path(testDir, "dep4.csv");
Appender appender4 = ((FileTablespace) TablespaceManager.getLocalFs())
.getAppender(dep4Meta, dep4Schema, dep4Path);
@@ -184,7 +185,7 @@
.build();
- TableMeta job3Meta = CatalogUtil.newTableMeta("TEXT");
+ TableMeta job3Meta = CatalogUtil.newTableMeta(BuiltinStorages.TEXT, util.getConfiguration());
Path job3Path = new Path(testDir, "job3.csv");
Appender appender2 = ((FileTablespace) TablespaceManager.getLocalFs())
.getAppender(job3Meta, job3Schema, job3Path);
@@ -225,7 +226,7 @@
.build();
- TableMeta emp3Meta = CatalogUtil.newTableMeta("TEXT");
+ TableMeta emp3Meta = CatalogUtil.newTableMeta(BuiltinStorages.TEXT, util.getConfiguration());
Path emp3Path = new Path(testDir, "emp3.csv");
Appender appender3 = ((FileTablespace) TablespaceManager.getLocalFs())
.getAppender(emp3Meta, emp3Schema, emp3Path);
@@ -278,7 +279,7 @@
.add("phone_number", Type.TEXT)
.build();
- TableMeta phone3Meta = CatalogUtil.newTableMeta("TEXT");
+ TableMeta phone3Meta = CatalogUtil.newTableMeta(BuiltinStorages.TEXT, util.getConfiguration());
Path phone3Path = new Path(testDir, "phone3.csv");
Appender appender5 = ((FileTablespace) TablespaceManager.getLocalFs())
.getAppender(phone3Meta, phone3Schema, phone3Path);
diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestSortExec.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestSortExec.java
index 07b392a..235d126 100644
--- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestSortExec.java
+++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestSortExec.java
@@ -19,10 +19,7 @@
package org.apache.tajo.engine.planner.physical;
import org.apache.hadoop.fs.Path;
-import org.apache.tajo.LocalTajoTestingUtility;
-import org.apache.tajo.TajoConstants;
-import org.apache.tajo.TajoTestingCluster;
-import org.apache.tajo.TpchTestBase;
+import org.apache.tajo.*;
import org.apache.tajo.algebra.Expr;
import org.apache.tajo.catalog.*;
import org.apache.tajo.common.TajoDataTypes.Type;
@@ -87,7 +84,7 @@
.add("deptname", Type.TEXT)
.build();
- employeeMeta = CatalogUtil.newTableMeta("TEXT");
+ employeeMeta = CatalogUtil.newTableMeta(BuiltinStorages.TEXT, util.getConfiguration());
tablePath = StorageUtil.concatPath(workDir, "employee", "table1");
sm.getFileSystem().mkdirs(tablePath.getParent());
diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestSortIntersectExec.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestSortIntersectExec.java
index 7a9385b..5200cde 100644
--- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestSortIntersectExec.java
+++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestSortIntersectExec.java
@@ -19,6 +19,7 @@
package org.apache.tajo.engine.planner.physical;
import org.apache.hadoop.fs.Path;
+import org.apache.tajo.BuiltinStorages;
import org.apache.tajo.LocalTajoTestingUtility;
import org.apache.tajo.TajoTestingCluster;
import org.apache.tajo.algebra.Expr;
@@ -91,7 +92,7 @@
.add("deptname", TajoDataTypes.Type.TEXT)
.build();
- TableMeta employeeMeta1 = CatalogUtil.newTableMeta("TEXT");
+ TableMeta employeeMeta1 = CatalogUtil.newTableMeta(BuiltinStorages.TEXT, util.getConfiguration());
Path employeePath1 = new Path(testDir, "employee1.csv");
Appender appender = ((FileTablespace) TablespaceManager.getLocalFs()).
getAppender(employeeMeta1, employeeSchema1, employeePath1);
@@ -119,7 +120,7 @@
.add("deptname", TajoDataTypes.Type.TEXT)
.build();
- TableMeta employeeMeta2 = CatalogUtil.newTableMeta("TEXT");
+ TableMeta employeeMeta2 = CatalogUtil.newTableMeta(BuiltinStorages.TEXT, util.getConfiguration());
Path employeePath2 = new Path(testDir, "employee2.csv");
Appender appender2 = ((FileTablespace) TablespaceManager.getLocalFs()).
getAppender(employeeMeta2, employeeSchema2, employeePath2);
diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestJsonWithTimezone.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestJsonWithTimezone.java
index 3380b1f..9eeba73 100644
--- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestJsonWithTimezone.java
+++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestJsonWithTimezone.java
@@ -18,7 +18,6 @@
package org.apache.tajo.engine.query;
-import com.google.common.collect.Lists;
import org.apache.tajo.QueryTestCaseBase;
import org.apache.tajo.SessionVars;
import org.junit.Test;
@@ -74,8 +73,6 @@
} finally {
executeString("DROP TABLE IF EXISTS timezoned3");
}
-
- getClient().unsetSessionVariables(Lists.newArrayList("TIMEZONE"));
}
@Test
@@ -96,8 +93,9 @@
@Test
public void testTimezonedTable5() throws Exception {
// Table - timezone = GMT+9 (by a specified system timezone)
- // TajoClient uses JVM default timezone (GMT+9)
+ // Client - GMT+9 (SET TIME ZONE 'GMT+9')
+ TimeZone systemTimeZone = testingCluster.getConfiguration().getSystemTimezone();
try {
testingCluster.getConfiguration().setSystemTimezone(TimeZone.getTimeZone("GMT+9"));
@@ -109,7 +107,7 @@
executeString("DROP TABLE IF EXISTS timezoned5");
// restore the config
- testingCluster.getConfiguration().setSystemTimezone(TimeZone.getTimeZone("GMT"));
+ testingCluster.getConfiguration().setSystemTimezone(systemTimeZone);
}
}
}
diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestSelectQuery.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestSelectQuery.java
index 61b5551..7a88198 100644
--- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestSelectQuery.java
+++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestSelectQuery.java
@@ -627,8 +627,6 @@
} finally {
executeString("DROP TABLE IF EXISTS timezoned3");
}
-
- getClient().unsetSessionVariables(Lists.newArrayList("TIMEZONE"));
}
@Test
@@ -649,8 +647,9 @@
@Test
public void testTimezonedTable5() throws Exception {
// Table - timezone = GMT+9 (by a specified system timezone)
- // TajoClient uses JVM default timezone (GMT+9)
+ // Client - GMT+9 (SET TIME ZONE 'GMT+9';)
+ TimeZone systemTimeZone = testingCluster.getConfiguration().getSystemTimezone();
try {
testingCluster.getConfiguration().setSystemTimezone(TimeZone.getTimeZone("GMT+9"));
@@ -662,7 +661,7 @@
executeString("DROP TABLE IF EXISTS timezoned5");
// restore the config
- testingCluster.getConfiguration().setSystemTimezone(TimeZone.getTimeZone("GMT"));
+ testingCluster.getConfiguration().setSystemTimezone(systemTimeZone);
}
}
@@ -677,7 +676,6 @@
ResultSet res = executeQuery();
assertResultSet(res, "testTimezonedTable3.result");
- executeString("SET TIME ZONE 'GMT'");
cleanupQuery(res);
} finally {
executeString("DROP TABLE IF EXISTS timezoned_load1");
diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestTimezone.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestTimezone.java
index 5034008..9c47d13 100644
--- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestTimezone.java
+++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestTimezone.java
@@ -19,7 +19,6 @@
package org.apache.tajo.engine.query;
import org.apache.tajo.QueryTestCaseBase;
-import org.apache.tajo.client.ResultSetUtil;
import org.apache.tajo.exception.TajoException;
import org.junit.Test;
import org.junit.runner.RunWith;
@@ -85,6 +84,28 @@
}
}
+ @Test
+ public void testCTASWithTimezone() throws TajoException, SQLException, IOException {
+ executeString(String.format("SET TIME ZONE TO '%s'", timezone)).close();
+ try {
+ executeString("create table test1 (col1 TIMESTAMP)").close();
+ executeString("insert overwrite into test1 select '2015-08-12 14:00:00'::TIMESTAMP").close();
+ try (ResultSet res = executeString("select * from test1")) {
+ assertTrue(res.next());
+ assertEquals("2015-08-12 14:00:00", res.getString(1));
+ }
+
+ executeString("create table test2 as select * from test1").close();
+ try (ResultSet res = executeString("select * from test2")) {
+ assertTrue(res.next());
+ assertEquals("2015-08-12 14:00:00", res.getString(1));
+ }
+ } finally {
+ executeString("drop table test1 purge").close();
+ executeString("drop table test2 purge").close();
+ }
+ }
+
@Parameters(name = "{index}: {0}")
public static Collection<Object []> getParameters() {
return Arrays.asList(new Object[][]{
diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/util/BenchmarkSort.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/util/BenchmarkSort.java
index d681d11..2c67a2d 100644
--- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/util/BenchmarkSort.java
+++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/util/BenchmarkSort.java
@@ -19,10 +19,7 @@
package org.apache.tajo.engine.util;
import org.apache.hadoop.fs.Path;
-import org.apache.tajo.LocalTajoTestingUtility;
-import org.apache.tajo.SessionVars;
-import org.apache.tajo.TajoConstants;
-import org.apache.tajo.TajoTestingCluster;
+import org.apache.tajo.*;
import org.apache.tajo.algebra.Expr;
import org.apache.tajo.catalog.*;
import org.apache.tajo.common.TajoDataTypes.Type;
@@ -42,10 +39,7 @@
import org.apache.tajo.plan.LogicalPlan;
import org.apache.tajo.plan.LogicalPlanner;
import org.apache.tajo.plan.logical.LogicalNode;
-import org.apache.tajo.storage.Appender;
-import org.apache.tajo.storage.FileTablespace;
-import org.apache.tajo.storage.TablespaceManager;
-import org.apache.tajo.storage.VTuple;
+import org.apache.tajo.storage.*;
import org.apache.tajo.storage.fragment.FileFragment;
import org.apache.tajo.util.CommonTestingUtil;
import org.apache.tajo.worker.TaskAttemptContext;
@@ -114,7 +108,7 @@
new Column("col14", Type.INT8),
}).build();
- TableMeta employeeMeta = CatalogUtil.newTableMeta("TEXT");
+ TableMeta employeeMeta = CatalogUtil.newTableMeta(BuiltinStorages.TEXT, conf);
Path employeePath = new Path(testDir, "employee.csv");
Appender appender = ((FileTablespace) TablespaceManager.getLocalFs())
.getAppender(employeeMeta, schema, employeePath);
diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/master/TestExecutionBlockCursor.java b/tajo-core-tests/src/test/java/org/apache/tajo/master/TestExecutionBlockCursor.java
index 7c6d892..56334ae 100644
--- a/tajo-core-tests/src/test/java/org/apache/tajo/master/TestExecutionBlockCursor.java
+++ b/tajo-core-tests/src/test/java/org/apache/tajo/master/TestExecutionBlockCursor.java
@@ -15,6 +15,7 @@
package org.apache.tajo.master;
import org.apache.hadoop.yarn.event.AsyncDispatcher;
+import org.apache.tajo.BuiltinStorages;
import org.apache.tajo.LocalTajoTestingUtility;
import org.apache.tajo.TajoTestingCluster;
import org.apache.tajo.algebra.Expr;
@@ -25,12 +26,12 @@
import org.apache.tajo.catalog.TableMeta;
import org.apache.tajo.catalog.statistics.TableStats;
import org.apache.tajo.conf.TajoConf;
-import org.apache.tajo.parser.sql.SQLAnalyzer;
import org.apache.tajo.engine.planner.global.ExecutionBlock;
import org.apache.tajo.engine.planner.global.ExecutionBlockCursor;
import org.apache.tajo.engine.planner.global.GlobalPlanner;
import org.apache.tajo.engine.planner.global.MasterPlan;
import org.apache.tajo.engine.query.QueryContext;
+import org.apache.tajo.parser.sql.SQLAnalyzer;
import org.apache.tajo.plan.LogicalOptimizer;
import org.apache.tajo.plan.LogicalPlan;
import org.apache.tajo.plan.LogicalPlanner;
@@ -69,7 +70,7 @@
tpch.loadSchemas();
tpch.loadOutSchema();
for (String table : tpch.getTableNames()) {
- TableMeta m = CatalogUtil.newTableMeta("TEXT");
+ TableMeta m = CatalogUtil.newTableMeta(BuiltinStorages.TEXT, util.getConfiguration());
TableDesc d = CatalogUtil.newTableDesc(
CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, table), tpch.getSchema(table), m, CommonTestingUtil.getTestDir());
TableStats stats = new TableStats();
diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/storage/TestQueryOnOrcFile.java b/tajo-core-tests/src/test/java/org/apache/tajo/storage/TestQueryOnOrcFile.java
index 29d132e..10f4265 100644
--- a/tajo-core-tests/src/test/java/org/apache/tajo/storage/TestQueryOnOrcFile.java
+++ b/tajo-core-tests/src/test/java/org/apache/tajo/storage/TestQueryOnOrcFile.java
@@ -47,7 +47,6 @@
executeString("SET TIME ZONE 'GMT+9'");
ResultSet res = executeQuery();
assertResultSet(res);
- executeString("SET TIME ZONE 'GMT'");
cleanupQuery(res);
}
@@ -56,7 +55,6 @@
executeString("SET TIME ZONE 'GMT+1'");
ResultSet res = executeString("select * from timezoned_orc");
assertResultSet(res);
- executeString("SET TIME ZONE 'GMT'");
cleanupQuery(res);
}
@@ -73,7 +71,6 @@
executeString("\\set TIMEZONE 'GMT-5'");
ResultSet res = executeString("select * from timezoned_orc");
assertResultSet(res);
- executeString("SET TIME ZONE 'GMT'");
cleanupQuery(res);
}
}
diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/storage/TestRowFile.java b/tajo-core-tests/src/test/java/org/apache/tajo/storage/TestRowFile.java
index cce0f4c..0173f59 100644
--- a/tajo-core-tests/src/test/java/org/apache/tajo/storage/TestRowFile.java
+++ b/tajo-core-tests/src/test/java/org/apache/tajo/storage/TestRowFile.java
@@ -69,7 +69,7 @@
.add("description", Type.TEXT)
.build();
- TableMeta meta = CatalogUtil.newTableMeta("ROWFILE");
+ TableMeta meta = CatalogUtil.newTableMeta("ROWFILE", conf);
FileTablespace sm = (FileTablespace) TablespaceManager.get(cluster.getDefaultFileSystem().getUri());
diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/worker/TestTaskExecutor.java b/tajo-core-tests/src/test/java/org/apache/tajo/worker/TestTaskExecutor.java
index df5b3c8..311f194 100644
--- a/tajo-core-tests/src/test/java/org/apache/tajo/worker/TestTaskExecutor.java
+++ b/tajo-core-tests/src/test/java/org/apache/tajo/worker/TestTaskExecutor.java
@@ -24,6 +24,7 @@
import org.apache.tajo.*;
import org.apache.tajo.catalog.statistics.TableStats;
import org.apache.tajo.conf.TajoConf;
+import org.apache.tajo.engine.query.QueryContext;
import org.apache.tajo.master.cluster.WorkerConnectionInfo;
import org.apache.tajo.rpc.CallFuture;
import org.apache.tajo.worker.event.NodeResourceAllocateEvent;
@@ -209,7 +210,8 @@
@Override
protected Task createTask(final ExecutionBlockContext context, TaskRequestProto taskRequest) {
final TaskAttemptId taskAttemptId = new TaskAttemptId(taskRequest.getId());
- final TaskAttemptContext taskAttemptContext = new TaskAttemptContext(null, context, taskAttemptId, null, null);
+ final TaskAttemptContext taskAttemptContext =
+ new TaskAttemptContext(new QueryContext(conf), context, taskAttemptId, null, null);
return new Task() {
@Override
diff --git a/tajo-core-tests/src/test/resources/results/TestJsonWithTimezone/testTimezonedTable2.result b/tajo-core-tests/src/test/resources/results/TestJsonWithTimezone/testTimezonedTable2.result
index 8b13e55..242e743 100644
--- a/tajo-core-tests/src/test/resources/results/TestJsonWithTimezone/testTimezonedTable2.result
+++ b/tajo-core-tests/src/test/resources/results/TestJsonWithTimezone/testTimezonedTable2.result
@@ -1,5 +1,5 @@
t_timestamp,t_time,t_date
-------------------------------
-1980-03-31 16:50:30.01,16:50:30.01,1980-04-01
-1980-03-31 16:50:30,16:50:30,1980-04-01
-1980-03-31 16:50:30,16:50:30,1980-04-01
\ No newline at end of file
+1980-03-31 16:50:30.01,01:50:30.01,1980-04-01
+1980-03-31 16:50:30,01:50:30,1980-04-01
+1980-03-31 16:50:30,01:50:30,1980-04-01
\ No newline at end of file
diff --git a/tajo-core-tests/src/test/resources/results/TestSelectQuery/testTimezonedTable2.result b/tajo-core-tests/src/test/resources/results/TestSelectQuery/testTimezonedTable2.result
index 8b13e55..242e743 100644
--- a/tajo-core-tests/src/test/resources/results/TestSelectQuery/testTimezonedTable2.result
+++ b/tajo-core-tests/src/test/resources/results/TestSelectQuery/testTimezonedTable2.result
@@ -1,5 +1,5 @@
t_timestamp,t_time,t_date
-------------------------------
-1980-03-31 16:50:30.01,16:50:30.01,1980-04-01
-1980-03-31 16:50:30,16:50:30,1980-04-01
-1980-03-31 16:50:30,16:50:30,1980-04-01
\ No newline at end of file
+1980-03-31 16:50:30.01,01:50:30.01,1980-04-01
+1980-03-31 16:50:30,01:50:30,1980-04-01
+1980-03-31 16:50:30,01:50:30,1980-04-01
\ No newline at end of file
diff --git a/tajo-core-tests/src/test/resources/results/TestTajoCli/testDescTable1.result b/tajo-core-tests/src/test/resources/results/TestTajoCli/testDescTable1.result
index f065e6e..a1baa09 100644
--- a/tajo-core-tests/src/test/resources/results/TestTajoCli/testDescTable1.result
+++ b/tajo-core-tests/src/test/resources/results/TestTajoCli/testDescTable1.result
@@ -5,7 +5,8 @@
store type: TEXT
number of rows: 0
volume: 0 B
-Options:
+Options:
+ 'timezone'='${table.timezone}'
'text.delimiter'='|'
schema:
@@ -19,7 +20,8 @@
store type: TEXT
number of rows: 0
volume: 0 B
-Options:
+Options:
+ 'timezone'='${table.timezone}'
'text.delimiter'='|'
schema:
diff --git a/tajo-core-tests/src/test/resources/results/TestTajoCli/testDescTable2.result b/tajo-core-tests/src/test/resources/results/TestTajoCli/testDescTable2.result
index 8534299..9c0abbf 100644
--- a/tajo-core-tests/src/test/resources/results/TestTajoCli/testDescTable2.result
+++ b/tajo-core-tests/src/test/resources/results/TestTajoCli/testDescTable2.result
@@ -5,7 +5,8 @@
store type: TEXT
number of rows: 0
volume: 0 B
-Options:
+Options:
+ 'timezone'='${table.timezone}'
'text.delimiter'='|'
schema:
@@ -19,9 +20,10 @@
store type: TEXT
number of rows: 0
volume: 0 B
-Options:
+Options:
+ 'timezone'='${table.timezone}'
'text.delimiter'='|'
schema:
col1 INT4
-col2 INT4
\ No newline at end of file
+col2 INT4
diff --git a/tajo-core-tests/src/test/resources/results/TestTajoCli/testDescTableForNestedSchema.result b/tajo-core-tests/src/test/resources/results/TestTajoCli/testDescTableForNestedSchema.result
index 29106b3..6337edf 100644
--- a/tajo-core-tests/src/test/resources/results/TestTajoCli/testDescTableForNestedSchema.result
+++ b/tajo-core-tests/src/test/resources/results/TestTajoCli/testDescTableForNestedSchema.result
@@ -5,7 +5,8 @@
store type: TEXT
number of rows: 0
volume: 0 B
-Options:
+Options:
+ 'timezone'='${table.timezone}'
'text.delimiter'='|'
schema:
@@ -20,7 +21,8 @@
store type: TEXT
number of rows: 0
volume: 0 B
-Options:
+Options:
+ 'timezone'='${table.timezone}'
'text.delimiter'='|'
schema:
diff --git a/tajo-core-tests/src/test/resources/results/TestTajoDump/testDump1.result b/tajo-core-tests/src/test/resources/results/TestTajoDump/testDump1.result
index 8accece..328b852 100644
--- a/tajo-core-tests/src/test/resources/results/TestTajoDump/testDump1.result
+++ b/tajo-core-tests/src/test/resources/results/TestTajoDump/testDump1.result
@@ -12,5 +12,5 @@
--
-- Name: "TestTajoDump"."TableName1"; Type: TABLE; Storage: TEXT
--
-CREATE TABLE "TestTajoDump"."TableName1" ("Age" INT4, "FirstName" TEXT, lastname TEXT) USING TEXT WITH ('text.delimiter'='|');
+CREATE TABLE "TestTajoDump"."TableName1" ("Age" INT4, "FirstName" TEXT, lastname TEXT) USING TEXT WITH ('text.delimiter'='|', 'timezone'='${table.timezone}');
diff --git a/tajo-core-tests/src/test/resources/results/TestTajoDump/testDump2.result b/tajo-core-tests/src/test/resources/results/TestTajoDump/testDump2.result
index 787562e..d84e830 100644
--- a/tajo-core-tests/src/test/resources/results/TestTajoDump/testDump2.result
+++ b/tajo-core-tests/src/test/resources/results/TestTajoDump/testDump2.result
@@ -12,5 +12,5 @@
--
-- Name: "TestTajoDump"."TableName2"; Type: TABLE; Storage: TEXT
--
-CREATE TABLE "TestTajoDump"."TableName2" ("Age" INT4, "Name" RECORD ("FirstName" TEXT, lastname TEXT)) USING TEXT WITH ('text.delimiter'='|');
+CREATE TABLE "TestTajoDump"."TableName2" ("Age" INT4, "Name" RECORD ("FirstName" TEXT, lastname TEXT)) USING TEXT WITH ('text.delimiter'='|', 'timezone'='${table.timezone}');
diff --git a/tajo-core-tests/src/test/resources/results/TestTajoDump/testDump3.result b/tajo-core-tests/src/test/resources/results/TestTajoDump/testDump3.result
index 1bde41d..b3cd13f 100644
--- a/tajo-core-tests/src/test/resources/results/TestTajoDump/testDump3.result
+++ b/tajo-core-tests/src/test/resources/results/TestTajoDump/testDump3.result
@@ -12,7 +12,7 @@
--
-- Name: "TestTajoDump"."TableName1"; Type: TABLE; Storage: TEXT
--
-CREATE TABLE "TestTajoDump"."TableName1" ("Age" INT4, "FirstName" TEXT, lastname TEXT) USING TEXT WITH ('text.delimiter'='|');
+CREATE TABLE "TestTajoDump"."TableName1" ("Age" INT4, "FirstName" TEXT, lastname TEXT) USING TEXT WITH ('text.delimiter'='|', 'timezone'='${table.timezone}');
--
-- Name: test_idx; Type: INDEX; Index Method: TWO_LEVEL_BIN_TREE
diff --git a/tajo-core-tests/src/test/resources/results/TestTajoDump/testPartitionsDump.result b/tajo-core-tests/src/test/resources/results/TestTajoDump/testPartitionsDump.result
index 677b5f2..e7b1ccd 100644
--- a/tajo-core-tests/src/test/resources/results/TestTajoDump/testPartitionsDump.result
+++ b/tajo-core-tests/src/test/resources/results/TestTajoDump/testPartitionsDump.result
@@ -12,7 +12,7 @@
--
-- Name: "TestTajoDump"."TableName3"; Type: TABLE; Storage: TEXT
--
-CREATE TABLE "TestTajoDump"."TableName3" (col1 INT4, col2 INT4) USING TEXT WITH ('text.delimiter'='|') PARTITION BY COLUMN(col3 INT4, col4 INT4);
+CREATE TABLE "TestTajoDump"."TableName3" (col1 INT4, col2 INT4) USING TEXT WITH ('text.delimiter'='|', 'timezone'='${table.timezone}') PARTITION BY COLUMN(col3 INT4, col4 INT4);
--
-- Table Partitions: TableName3
@@ -24,7 +24,7 @@
--
-- Name: "TestTajoDump"."TableName4"; Type: TABLE; Storage: TEXT
--
-CREATE TABLE "TestTajoDump"."TableName4" (col1 INT4, col2 INT4) USING TEXT WITH ('text.delimiter'='|') PARTITION BY COLUMN(col3 TEXT, col4 DATE);
+CREATE TABLE "TestTajoDump"."TableName4" (col1 INT4, col2 INT4) USING TEXT WITH ('text.delimiter'='|', 'timezone'='${table.timezone}') PARTITION BY COLUMN(col3 TEXT, col4 DATE);
--
-- Table Partitions: TableName4
diff --git a/tajo-core-tests/src/test/resources/results/testDDLBuilder/testBuildDDLForBaseTable.result b/tajo-core-tests/src/test/resources/results/testDDLBuilder/testBuildDDLForBaseTable.result
index 7e10a3b..c635e5d 100644
--- a/tajo-core-tests/src/test/resources/results/testDDLBuilder/testBuildDDLForBaseTable.result
+++ b/tajo-core-tests/src/test/resources/results/testDDLBuilder/testBuildDDLForBaseTable.result
@@ -1,4 +1,4 @@
--
-- Name: db1.table2; Type: TABLE; Storage: TEXT
--
-CREATE TABLE db1.table2 (name BLOB, addr TEXT) USING TEXT WITH ('compression.codec'='org.apache.hadoop.io.compress.GzipCodec', 'text.delimiter'='|');
\ No newline at end of file
+CREATE TABLE db1.table2 (name BLOB, addr TEXT) USING TEXT WITH ('compression.codec'='org.apache.hadoop.io.compress.GzipCodec', 'text.delimiter'='|', 'timezone'='Asia/Seoul');
\ No newline at end of file
diff --git a/tajo-core-tests/src/test/resources/results/testDDLBuilder/testBuildDDLForExternalTable.result b/tajo-core-tests/src/test/resources/results/testDDLBuilder/testBuildDDLForExternalTable.result
index 535bc11..011dfc1 100644
--- a/tajo-core-tests/src/test/resources/results/testDDLBuilder/testBuildDDLForExternalTable.result
+++ b/tajo-core-tests/src/test/resources/results/testDDLBuilder/testBuildDDLForExternalTable.result
@@ -2,4 +2,4 @@
-- Name: db1.table1; Type: TABLE; Storage: TEXT
-- Path: /table1
--
-CREATE EXTERNAL TABLE db1.table1 (name BLOB, addr TEXT) USING TEXT WITH ('compression.codec'='org.apache.hadoop.io.compress.GzipCodec', 'text.delimiter'='|') PARTITION BY COLUMN(key INT4, key2 TEXT) LOCATION '/table1';
\ No newline at end of file
+CREATE EXTERNAL TABLE db1.table1 (name BLOB, addr TEXT) USING TEXT WITH ('compression.codec'='org.apache.hadoop.io.compress.GzipCodec', 'text.delimiter'='|', 'timezone'='Asia/Seoul') PARTITION BY COLUMN(key INT4, key2 TEXT) LOCATION '/table1';
\ No newline at end of file
diff --git a/tajo-core-tests/src/test/resources/results/testDDLBuilder/testBuildDDLQuotedTableName1.result b/tajo-core-tests/src/test/resources/results/testDDLBuilder/testBuildDDLQuotedTableName1.result
index 34b6fb3..a432033 100644
--- a/tajo-core-tests/src/test/resources/results/testDDLBuilder/testBuildDDLQuotedTableName1.result
+++ b/tajo-core-tests/src/test/resources/results/testDDLBuilder/testBuildDDLQuotedTableName1.result
@@ -2,4 +2,4 @@
-- Name: db1."TABLE2"; Type: TABLE; Storage: TEXT
-- Path: /table1
--
-CREATE EXTERNAL TABLE db1."TABLE2" (name BLOB, addr TEXT, "FirstName" TEXT, "LastName" TEXT, "with" TEXT) USING TEXT WITH ('compression.codec'='org.apache.hadoop.io.compress.GzipCodec', 'text.delimiter'='|') PARTITION BY COLUMN("BirthYear" INT4) LOCATION '/table1';
\ No newline at end of file
+CREATE EXTERNAL TABLE db1."TABLE2" (name BLOB, addr TEXT, "FirstName" TEXT, "LastName" TEXT, "with" TEXT) USING TEXT WITH ('compression.codec'='org.apache.hadoop.io.compress.GzipCodec', 'text.delimiter'='|', 'timezone'='Asia/Seoul') PARTITION BY COLUMN("BirthYear" INT4) LOCATION '/table1';
\ No newline at end of file
diff --git a/tajo-core-tests/src/test/resources/results/testDDLBuilder/testBuildDDLQuotedTableName2.result b/tajo-core-tests/src/test/resources/results/testDDLBuilder/testBuildDDLQuotedTableName2.result
index cd1ebf8..2e175de 100644
--- a/tajo-core-tests/src/test/resources/results/testDDLBuilder/testBuildDDLQuotedTableName2.result
+++ b/tajo-core-tests/src/test/resources/results/testDDLBuilder/testBuildDDLQuotedTableName2.result
@@ -1,4 +1,4 @@
--
-- Name: db1."TABLE1"; Type: TABLE; Storage: TEXT
--
-CREATE TABLE db1."TABLE1" (name BLOB, addr TEXT, "FirstName" TEXT, "LastName" TEXT, "with" TEXT) USING TEXT WITH ('compression.codec'='org.apache.hadoop.io.compress.GzipCodec', 'text.delimiter'='|') PARTITION BY COLUMN("BirthYear" INT4);
\ No newline at end of file
+CREATE TABLE db1."TABLE1" (name BLOB, addr TEXT, "FirstName" TEXT, "LastName" TEXT, "with" TEXT) USING TEXT WITH ('compression.codec'='org.apache.hadoop.io.compress.GzipCodec', 'text.delimiter'='|', 'timezone'='Asia/Seoul') PARTITION BY COLUMN("BirthYear" INT4);
\ No newline at end of file
diff --git a/tajo-core/src/main/java/org/apache/tajo/benchmark/TPCH.java b/tajo-core/src/main/java/org/apache/tajo/benchmark/TPCH.java
index 3ff773b..87e1b54 100644
--- a/tajo-core/src/main/java/org/apache/tajo/benchmark/TPCH.java
+++ b/tajo-core/src/main/java/org/apache/tajo/benchmark/TPCH.java
@@ -22,6 +22,7 @@
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.fs.Path;
+import org.apache.tajo.BuiltinStorages;
import org.apache.tajo.catalog.CatalogUtil;
import org.apache.tajo.catalog.Schema;
import org.apache.tajo.catalog.SchemaBuilder;
@@ -30,8 +31,8 @@
import org.apache.tajo.catalog.proto.CatalogProtos;
import org.apache.tajo.common.TajoDataTypes;
import org.apache.tajo.common.TajoDataTypes.Type;
+import org.apache.tajo.conf.TajoConf;
import org.apache.tajo.exception.TajoException;
-import org.apache.tajo.storage.StorageConstants;
import java.io.IOException;
import java.util.Map;
@@ -211,8 +212,7 @@
}
public void loadTable(String tableName) throws TajoException {
- TableMeta meta = CatalogUtil.newTableMeta("TEXT");
- meta.putProperty(StorageConstants.TEXT_DELIMITER, StorageConstants.DEFAULT_FIELD_DELIMITER);
+ TableMeta meta = CatalogUtil.newTableMeta(BuiltinStorages.TEXT, new TajoConf());
PartitionMethodDesc partitionMethodDesc = null;
if (tableName.equals(CUSTOMER_PARTS)) {
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/datetime/AddDays.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/datetime/AddDays.java
index 5543024..4543441 100644
--- a/tajo-core/src/main/java/org/apache/tajo/engine/function/datetime/AddDays.java
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/datetime/AddDays.java
@@ -18,16 +18,22 @@
package org.apache.tajo.engine.function.datetime;
+import org.apache.tajo.OverridableConf;
+import org.apache.tajo.SessionVars;
import org.apache.tajo.catalog.Column;
import org.apache.tajo.common.TajoDataTypes;
import org.apache.tajo.common.TajoDataTypes.Type;
import org.apache.tajo.datum.Datum;
+import org.apache.tajo.datum.DatumFactory;
import org.apache.tajo.datum.IntervalDatum;
-import org.apache.tajo.plan.function.GeneralFunction;
import org.apache.tajo.engine.function.annotation.Description;
import org.apache.tajo.engine.function.annotation.ParamTypes;
+import org.apache.tajo.plan.expr.FunctionEval;
+import org.apache.tajo.plan.function.GeneralFunction;
import org.apache.tajo.storage.Tuple;
+import java.util.TimeZone;
+
@Description(
functionName = "add_days",
description = "Return date value which is added with given parameter.",
@@ -44,6 +50,7 @@
}
)
public class AddDays extends GeneralFunction {
+
public AddDays() {
super(new Column[]{
new Column("date", TajoDataTypes.Type.DATE),
@@ -52,8 +59,14 @@
}
@Override
+ public void init(OverridableConf context, FunctionEval.ParamType[] types) {
+ setTimeZone(TimeZone.getTimeZone(context.get(SessionVars.TIMEZONE)));
+ }
+
+ @Override
public Datum eval(Tuple params) {
- Datum dateDatum = params.asDatum(0);
+ // cast to UTC timestamp
+ Datum dateDatum = DatumFactory.createTimestamp(params.asDatum(0), getTimeZone());
long val = params.getInt8(1);
if (val >= 0) {
return dateDatum.plus(new IntervalDatum(val * IntervalDatum.DAY_MILLIS));
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/datetime/AddMonths.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/datetime/AddMonths.java
index 016a25c..80292ca 100644
--- a/tajo-core/src/main/java/org/apache/tajo/engine/function/datetime/AddMonths.java
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/datetime/AddMonths.java
@@ -18,16 +18,22 @@
package org.apache.tajo.engine.function.datetime;
+import org.apache.tajo.OverridableConf;
+import org.apache.tajo.SessionVars;
import org.apache.tajo.catalog.Column;
import org.apache.tajo.common.TajoDataTypes;
import org.apache.tajo.common.TajoDataTypes.Type;
import org.apache.tajo.datum.Datum;
+import org.apache.tajo.datum.DatumFactory;
import org.apache.tajo.datum.IntervalDatum;
-import org.apache.tajo.plan.function.GeneralFunction;
import org.apache.tajo.engine.function.annotation.Description;
import org.apache.tajo.engine.function.annotation.ParamTypes;
+import org.apache.tajo.plan.expr.FunctionEval;
+import org.apache.tajo.plan.function.GeneralFunction;
import org.apache.tajo.storage.Tuple;
+import java.util.TimeZone;
+
@Description(
functionName = "add_months",
description = "Return date value which is added with given parameter.",
@@ -44,6 +50,7 @@
}
)
public class AddMonths extends GeneralFunction {
+
public AddMonths() {
super(new Column[]{
new Column("date", TajoDataTypes.Type.DATE),
@@ -52,8 +59,15 @@
}
@Override
+ public void init(OverridableConf context, FunctionEval.ParamType[] types) {
+ setTimeZone(TimeZone.getTimeZone(context.get(SessionVars.TIMEZONE)));
+ }
+
+ @Override
public Datum eval(Tuple params) {
- Datum dateDatum = params.asDatum(0);
+ // cast to UTC timestamp
+ Datum dateDatum = DatumFactory.createTimestamp(params.asDatum(0), getTimeZone());
+
int val = params.getInt4(1);
if (val >= 0) {
return dateDatum.plus(new IntervalDatum(val, 0));
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/datetime/CurrentDate.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/datetime/CurrentDate.java
index d57ac02..378ae3f 100644
--- a/tajo-core/src/main/java/org/apache/tajo/engine/function/datetime/CurrentDate.java
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/datetime/CurrentDate.java
@@ -18,10 +18,8 @@
package org.apache.tajo.engine.function.datetime;
-import com.google.gson.annotations.Expose;
import org.apache.tajo.OverridableConf;
import org.apache.tajo.SessionVars;
-import org.apache.tajo.TajoConstants;
import org.apache.tajo.common.TajoDataTypes;
import org.apache.tajo.datum.DateDatum;
import org.apache.tajo.datum.Datum;
@@ -44,7 +42,6 @@
paramTypes = {@ParamTypes(paramTypes = {})}
)
public class CurrentDate extends GeneralFunction {
- @Expose private TimeZone timezone;
private DateDatum datum;
public CurrentDate() {
@@ -53,8 +50,9 @@
@Override
public void init(OverridableConf context, FunctionEval.ParamType[] types) {
- String timezoneId = context.get(SessionVars.TIMEZONE, TajoConstants.DEFAULT_SYSTEM_TIMEZONE);
- timezone = TimeZone.getTimeZone(timezoneId);
+ if (!hasTimeZone()) {
+ setTimeZone(TimeZone.getTimeZone(context.get(SessionVars.TIMEZONE)));
+ }
}
@Override
@@ -63,7 +61,7 @@
long julianTimestamp = DateTimeUtil.javaTimeToJulianTime(System.currentTimeMillis());
TimeMeta tm = new TimeMeta();
DateTimeUtil.toJulianTimeMeta(julianTimestamp, tm);
- DateTimeUtil.toUserTimezone(tm, timezone);
+ DateTimeUtil.toUserTimezone(tm, getTimeZone());
datum = DatumFactory.createDate(tm.years, tm.monthOfYear, tm.dayOfMonth);
}
return datum;
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/datetime/CurrentTime.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/datetime/CurrentTime.java
index 0ba5ca0..1e8eb2d 100644
--- a/tajo-core/src/main/java/org/apache/tajo/engine/function/datetime/CurrentTime.java
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/datetime/CurrentTime.java
@@ -18,17 +18,22 @@
package org.apache.tajo.engine.function.datetime;
+import org.apache.tajo.OverridableConf;
+import org.apache.tajo.SessionVars;
import org.apache.tajo.common.TajoDataTypes;
import org.apache.tajo.datum.Datum;
import org.apache.tajo.datum.DatumFactory;
import org.apache.tajo.datum.TimeDatum;
-import org.apache.tajo.plan.function.GeneralFunction;
import org.apache.tajo.engine.function.annotation.Description;
import org.apache.tajo.engine.function.annotation.ParamTypes;
+import org.apache.tajo.plan.expr.FunctionEval;
+import org.apache.tajo.plan.function.GeneralFunction;
import org.apache.tajo.storage.Tuple;
import org.apache.tajo.util.datetime.DateTimeUtil;
import org.apache.tajo.util.datetime.TimeMeta;
+import java.util.TimeZone;
+
@Description(
functionName = "current_time",
description = "Get current time. Result is TIME type.",
@@ -44,11 +49,19 @@
}
@Override
+ public void init(OverridableConf context, FunctionEval.ParamType [] types) {
+ if (!hasTimeZone()) {
+ setTimeZone(TimeZone.getTimeZone(context.get(SessionVars.TIMEZONE)));
+ }
+ }
+
+ @Override
public Datum eval(Tuple params) {
if (datum == null) {
long julianTimestamp = DateTimeUtil.javaTimeToJulianTime(System.currentTimeMillis());
TimeMeta tm = new TimeMeta();
DateTimeUtil.toJulianTimeMeta(julianTimestamp, tm);
+ DateTimeUtil.toUserTimezone(tm, getTimeZone());
datum = DatumFactory.createTime(DateTimeUtil.toTime(tm));
}
return datum;
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/datetime/DatePartFromTime.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/datetime/DatePartFromTime.java
index 8848af6..3911ae8 100644
--- a/tajo-core/src/main/java/org/apache/tajo/engine/function/datetime/DatePartFromTime.java
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/datetime/DatePartFromTime.java
@@ -18,10 +18,7 @@
package org.apache.tajo.engine.function.datetime;
-import com.google.gson.annotations.Expose;
import org.apache.tajo.OverridableConf;
-import org.apache.tajo.SessionVars;
-import org.apache.tajo.TajoConstants;
import org.apache.tajo.catalog.Column;
import org.apache.tajo.common.TajoDataTypes;
import org.apache.tajo.datum.Datum;
@@ -33,14 +30,9 @@
import org.apache.tajo.plan.function.GeneralFunction;
import org.apache.tajo.storage.Tuple;
import org.apache.tajo.util.datetime.DateTimeConstants;
-import org.apache.tajo.util.datetime.DateTimeUtil;
import org.apache.tajo.util.datetime.TimeMeta;
-import java.util.TimeZone;
-
-import static org.apache.tajo.common.TajoDataTypes.Type.TIME;
-import static org.apache.tajo.common.TajoDataTypes.Type.FLOAT8;
-import static org.apache.tajo.common.TajoDataTypes.Type.TEXT;
+import static org.apache.tajo.common.TajoDataTypes.Type.*;
@Description(
functionName = "date_part",
@@ -51,7 +43,6 @@
paramTypes = {@ParamTypes(paramTypes = {TajoDataTypes.Type.TEXT, TajoDataTypes.Type.TIME})}
)
public class DatePartFromTime extends GeneralFunction {
- @Expose private TimeZone timezone;
private DatePartExtractorFromTime extractor = null;
public DatePartFromTime() {
@@ -63,8 +54,6 @@
@Override
public void init(OverridableConf context, FunctionEval.ParamType [] types) {
- String timezoneId = context.get(SessionVars.TIMEZONE, TajoConstants.DEFAULT_SYSTEM_TIMEZONE);
- timezone = TimeZone.getTimeZone(timezoneId);
}
@Override
@@ -93,7 +82,6 @@
}
TimeMeta tm = params.getTimeDate(1);
- DateTimeUtil.toUserTimezone(tm, timezone);
return extractor.extract(tm);
}
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/datetime/DatePartFromTimestamp.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/datetime/DatePartFromTimestamp.java
index 05fa8e4..48ca7b2 100644
--- a/tajo-core/src/main/java/org/apache/tajo/engine/function/datetime/DatePartFromTimestamp.java
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/datetime/DatePartFromTimestamp.java
@@ -18,17 +18,17 @@
package org.apache.tajo.engine.function.datetime;
-import com.google.gson.annotations.Expose;
import org.apache.tajo.OverridableConf;
import org.apache.tajo.SessionVars;
-import org.apache.tajo.TajoConstants;
import org.apache.tajo.catalog.Column;
import org.apache.tajo.common.TajoDataTypes;
-import org.apache.tajo.datum.*;
-import org.apache.tajo.plan.expr.FunctionEval;
-import org.apache.tajo.plan.function.GeneralFunction;
+import org.apache.tajo.datum.Datum;
+import org.apache.tajo.datum.DatumFactory;
+import org.apache.tajo.datum.NullDatum;
import org.apache.tajo.engine.function.annotation.Description;
import org.apache.tajo.engine.function.annotation.ParamTypes;
+import org.apache.tajo.plan.expr.FunctionEval;
+import org.apache.tajo.plan.function.GeneralFunction;
import org.apache.tajo.storage.Tuple;
import org.apache.tajo.util.datetime.DateTimeConstants;
import org.apache.tajo.util.datetime.DateTimeUtil;
@@ -47,7 +47,6 @@
paramTypes = {@ParamTypes(paramTypes = {TajoDataTypes.Type.TEXT, TajoDataTypes.Type.TIMESTAMP})}
)
public class DatePartFromTimestamp extends GeneralFunction {
- @Expose private TimeZone timezone;
private DatePartExtractorFromTimestamp extractor = null;
public DatePartFromTimestamp() {
@@ -59,8 +58,7 @@
@Override
public void init(OverridableConf context, FunctionEval.ParamType [] types) {
- String timezoneId = context.get(SessionVars.TIMEZONE, TajoConstants.DEFAULT_SYSTEM_TIMEZONE);
- timezone = TimeZone.getTimeZone(timezoneId);
+ setTimeZone(TimeZone.getTimeZone(context.get(SessionVars.TIMEZONE)));
}
@Override
@@ -121,7 +119,7 @@
}
TimeMeta tm = params.getTimeDate(1);
- DateTimeUtil.toUserTimezone(tm, timezone);
+ DateTimeUtil.toUserTimezone(tm, getTimeZone());
return extractor.extract(tm);
}
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/datetime/ToCharTimestamp.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/datetime/ToCharTimestamp.java
index 7b620e9..39f112d 100644
--- a/tajo-core/src/main/java/org/apache/tajo/engine/function/datetime/ToCharTimestamp.java
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/datetime/ToCharTimestamp.java
@@ -18,10 +18,7 @@
package org.apache.tajo.engine.function.datetime;
-import com.google.gson.annotations.Expose;
import org.apache.tajo.OverridableConf;
-import org.apache.tajo.SessionVars;
-import org.apache.tajo.TajoConstants;
import org.apache.tajo.catalog.Column;
import org.apache.tajo.common.TajoDataTypes;
import org.apache.tajo.datum.Datum;
@@ -36,8 +33,6 @@
import org.apache.tajo.util.datetime.DateTimeUtil;
import org.apache.tajo.util.datetime.TimeMeta;
-import java.util.TimeZone;
-
import static org.apache.tajo.common.TajoDataTypes.Type.TEXT;
import static org.apache.tajo.common.TajoDataTypes.Type.TIMESTAMP;
@@ -50,7 +45,6 @@
paramTypes = {@ParamTypes(paramTypes = {TajoDataTypes.Type.TIMESTAMP, TajoDataTypes.Type.TEXT})}
)
public class ToCharTimestamp extends GeneralFunction {
- @Expose private TimeZone timezone;
public ToCharTimestamp() {
super(new Column[] {
@@ -61,8 +55,9 @@
@Override
public void init(OverridableConf context, FunctionEval.ParamType[] paramTypes) {
- String timezoneId = context.get(SessionVars.TIMEZONE, TajoConstants.DEFAULT_SYSTEM_TIMEZONE);
- timezone = TimeZone.getTimeZone(timezoneId);
+ if (!hasTimeZone()) {
+ setTimeZone(context.getConf().getSystemTimezone());
+ }
}
@Override
@@ -74,7 +69,7 @@
TimeMeta tm = params.getTimeDate(0);
String pattern = params.getText(1);
- DateTimeUtil.toUserTimezone(tm, timezone);
+ DateTimeUtil.toUserTimezone(tm, getTimeZone());
return DatumFactory.createText(DateTimeFormat.to_char(tm, pattern));
}
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/datetime/ToTimestampText.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/datetime/ToTimestampText.java
index ff92795..5201f33 100644
--- a/tajo-core/src/main/java/org/apache/tajo/engine/function/datetime/ToTimestampText.java
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/datetime/ToTimestampText.java
@@ -19,22 +19,20 @@
package org.apache.tajo.engine.function.datetime;
import org.apache.tajo.OverridableConf;
-import org.apache.tajo.SessionVars;
-import org.apache.tajo.TajoConstants;
import org.apache.tajo.catalog.Column;
import org.apache.tajo.common.TajoDataTypes;
-import org.apache.tajo.datum.*;
-import org.apache.tajo.plan.expr.FunctionEval;
-import org.apache.tajo.plan.function.GeneralFunction;
+import org.apache.tajo.datum.Datum;
+import org.apache.tajo.datum.NullDatum;
+import org.apache.tajo.datum.TimestampDatum;
import org.apache.tajo.engine.function.annotation.Description;
import org.apache.tajo.engine.function.annotation.ParamTypes;
+import org.apache.tajo.plan.expr.FunctionEval;
+import org.apache.tajo.plan.function.GeneralFunction;
import org.apache.tajo.storage.Tuple;
import org.apache.tajo.util.datetime.DateTimeFormat;
import org.apache.tajo.util.datetime.DateTimeUtil;
import org.apache.tajo.util.datetime.TimeMeta;
-import java.util.TimeZone;
-
import static org.apache.tajo.common.TajoDataTypes.Type.TEXT;
@Description(
@@ -47,15 +45,15 @@
paramTypes = {@ParamTypes(paramTypes = {TajoDataTypes.Type.TEXT, TajoDataTypes.Type.TEXT})}
)
public class ToTimestampText extends GeneralFunction {
- private TimeZone timezone;
public ToTimestampText() {
super(new Column[]{new Column("DateTimeText", TEXT), new Column("Pattern", TEXT)});
}
- public void init(OverridableConf queryContext, FunctionEval.ParamType [] paramTypes) {
- String timezoneId = queryContext.get(SessionVars.TIMEZONE, TajoConstants.DEFAULT_SYSTEM_TIMEZONE);
- timezone = TimeZone.getTimeZone(timezoneId);
+ public void init(OverridableConf context, FunctionEval.ParamType [] paramTypes) {
+ if (!hasTimeZone()) {
+ setTimeZone(context.getConf().getSystemTimezone());
+ }
}
@Override
@@ -65,7 +63,7 @@
}
TimeMeta tm = DateTimeFormat.parseDateTime(params.getText(0), params.getText(1));
- DateTimeUtil.toUTCTimezone(tm, timezone);
+ DateTimeUtil.toUTCTimezone(tm, getTimeZone());
return new TimestampDatum(DateTimeUtil.toJulianTimestamp(tm));
}
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/ColPartitionStoreExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/ColPartitionStoreExec.java
index 83a9ff8..0286340 100644
--- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/ColPartitionStoreExec.java
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/ColPartitionStoreExec.java
@@ -75,7 +75,7 @@
if (this.plan.hasOptions()) {
meta = CatalogUtil.newTableMeta(plan.getStorageType(), plan.getOptions());
} else {
- meta = CatalogUtil.newTableMeta(plan.getStorageType());
+ meta = CatalogUtil.newTableMeta(plan.getStorageType(), context.getConf());
}
PhysicalPlanUtil.setNullCharIfNecessary(context.getQueryContext(), plan, meta);
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/ExternalSortExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/ExternalSortExec.java
index e269bf6..ae16667 100644
--- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/ExternalSortExec.java
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/ExternalSortExec.java
@@ -138,7 +138,7 @@
this.allocatedCoreNum = context.getConf().getIntVar(ConfVars.EXECUTOR_EXTERNAL_SORT_THREAD_NUM);
this.localDirAllocator = new LocalDirAllocator(ConfVars.WORKER_TEMPORAL_DIR.varname);
this.localFS = new RawLocalFileSystem();
- this.intermediateMeta = CatalogUtil.newTableMeta(BuiltinStorages.DRAW);
+ this.intermediateMeta = CatalogUtil.newTableMeta(BuiltinStorages.DRAW, context.getConf());
this.inputStats = new TableStats();
this.sortAlgorithm = getSortAlgorithm(context.getQueryContext(), sortSpecs);
LOG.info(sortAlgorithm.name() + " sort is selected");
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/HashShuffleFileWriteExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/HashShuffleFileWriteExec.java
index 5563ab9..0da1aa6 100644
--- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/HashShuffleFileWriteExec.java
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/HashShuffleFileWriteExec.java
@@ -84,7 +84,7 @@
if (plan.hasOptions()) {
this.meta = CatalogUtil.newTableMeta(plan.getStorageType(), plan.getOptions());
} else {
- this.meta = CatalogUtil.newTableMeta(plan.getStorageType());
+ this.meta = CatalogUtil.newTableMeta(plan.getStorageType(), context.getConf());
}
// about the shuffle
this.numShuffleOutputs = this.plan.getNumOutputs();
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/InsertRowsExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/InsertRowsExec.java
index 6c2f7a5..9648002 100644
--- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/InsertRowsExec.java
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/InsertRowsExec.java
@@ -58,7 +58,7 @@
if (plan.hasOptions()) {
meta = CatalogUtil.newTableMeta(plan.getStorageType(), plan.getOptions());
} else {
- meta = CatalogUtil.newTableMeta(plan.getStorageType());
+ meta = CatalogUtil.newTableMeta(plan.getStorageType(), context.getConf());
}
PhysicalPlanUtil.setNullCharIfNecessary(context.getQueryContext(), plan, meta);
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/RangeShuffleFileWriteExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/RangeShuffleFileWriteExec.java
index 776a783..c7e5bec 100644
--- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/RangeShuffleFileWriteExec.java
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/RangeShuffleFileWriteExec.java
@@ -64,7 +64,7 @@
if (plan.hasOptions()) {
this.meta = CatalogUtil.newTableMeta(plan.getStorageType(), plan.getOptions());
} else {
- this.meta = CatalogUtil.newTableMeta(plan.getStorageType());
+ this.meta = CatalogUtil.newTableMeta(plan.getStorageType(), context.getConf());
}
}
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/StoreTableExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/StoreTableExec.java
index 2ebad1e..08ecb95 100644
--- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/StoreTableExec.java
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/StoreTableExec.java
@@ -66,7 +66,7 @@
if (plan.hasOptions()) {
meta = CatalogUtil.newTableMeta(plan.getStorageType(), plan.getOptions());
} else {
- meta = CatalogUtil.newTableMeta(plan.getStorageType());
+ meta = CatalogUtil.newTableMeta(plan.getStorageType(), context.getConf());
}
PhysicalPlanUtil.setNullCharIfNecessary(context.getQueryContext(), plan, meta);
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/exec/CreateTableExecutor.java b/tajo-core/src/main/java/org/apache/tajo/master/exec/CreateTableExecutor.java
index a2b5ab9..55d5442 100644
--- a/tajo-core/src/main/java/org/apache/tajo/master/exec/CreateTableExecutor.java
+++ b/tajo-core/src/main/java/org/apache/tajo/master/exec/CreateTableExecutor.java
@@ -57,7 +57,7 @@
if (createTable.hasOptions()) {
meta = CatalogUtil.newTableMeta(createTable.getStorageType(), createTable.getOptions());
} else {
- meta = CatalogUtil.newTableMeta(createTable.getStorageType());
+ meta = CatalogUtil.newTableMeta(createTable.getStorageType(), queryContext.getConf());
}
if(PlannerUtil.isFileStorageType(createTable.getStorageType()) && createTable.isExternal()){
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/exec/QueryExecutor.java b/tajo-core/src/main/java/org/apache/tajo/master/exec/QueryExecutor.java
index 1a51d98..f0a6b76 100644
--- a/tajo-core/src/main/java/org/apache/tajo/master/exec/QueryExecutor.java
+++ b/tajo-core/src/main/java/org/apache/tajo/master/exec/QueryExecutor.java
@@ -73,6 +73,7 @@
import java.util.ArrayList;
import java.util.List;
import java.util.Optional;
+import java.util.TimeZone;
import static org.apache.tajo.exception.ReturnStateUtil.OK;
import static org.apache.tajo.exception.ReturnStateUtil.errUndefinedDatabase;
@@ -327,6 +328,11 @@
LogicalRootNode rootNode = plan.getRootBlock().getRoot();
EvalContext evalContext = new EvalContext();
+
+ //Non From query should be session's time zone. e,g, select to_char(now(), 'yyyy-MM-dd')
+ String timezoneId = queryContext.get(SessionVars.TIMEZONE);
+ evalContext.setTimeZone(TimeZone.getTimeZone(timezoneId));
+
List<Target> targets = plan.getRootBlock().getRawTargets();
if (targets == null) {
throw new TajoInternalError("no targets");
diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/TaskAttemptContext.java b/tajo-core/src/main/java/org/apache/tajo/worker/TaskAttemptContext.java
index d56b6b4..47f1af2 100644
--- a/tajo-core/src/main/java/org/apache/tajo/worker/TaskAttemptContext.java
+++ b/tajo-core/src/main/java/org/apache/tajo/worker/TaskAttemptContext.java
@@ -120,6 +120,9 @@
this.partitionOutputVolume = Maps.newHashMap();
this.partitions = new ArrayList<>();
+
+ // set to system timezone for forwarded query
+ this.evalContext.setTimeZone(queryContext.getConf().getSystemTimezone());
}
@VisibleForTesting
diff --git a/tajo-docs/src/main/sphinx/jdbc_driver.rst b/tajo-docs/src/main/sphinx/jdbc_driver.rst
index 176477a..41ad6f2 100644
--- a/tajo-docs/src/main/sphinx/jdbc_driver.rst
+++ b/tajo-docs/src/main/sphinx/jdbc_driver.rst
@@ -83,6 +83,7 @@
* ``connectTimeout = int (seconds)`` - The timeout value used for socket connect operations. If connecting to the server takes longer than this value, the connection is broken. The timeout is specified in seconds and a value of zero means that it is disabled.
* ``socketTimeout = int (seconds)`` - The timeout value used for socket read operations. If reading from the server takes longer than this value, the connection is closed. This can be used as both a brute force global query timeout and a method of detecting network problems. The timeout is specified in seconds and a value of zero means that it is disabled.
* ``retry = int`` - Number of retry operation. Tajo JDBC driver is resilient against some network or connection problems. It determines how many times the connection will retry.
+ * ``timezone = string (timezone id e,g, 'Asia/Tokyo')`` - Each connection has its own client time zone setting.
An Example JDBC Client
diff --git a/tajo-docs/src/main/sphinx/time_zone.rst b/tajo-docs/src/main/sphinx/time_zone.rst
index 9b86e18..1d6fb27 100644
--- a/tajo-docs/src/main/sphinx/time_zone.rst
+++ b/tajo-docs/src/main/sphinx/time_zone.rst
@@ -2,7 +2,7 @@
Time Zone
******************
-Time zone affects some data types (e.g., ``Timestamp`` and ``Time``) and operations (e.g., ``to_char``). Tables can have different time zones. Internally, Tajo translates all table rows to UTC values and processes them. It becomes easier for Tajo to handle multiple different time zones.
+Time zone affects ``Timestamp`` data type and operations (e.g., ``to_char``). Tables can have different time zones. Internally, Tajo translates all table rows to UTC values and processes them. It becomes easier for Tajo to handle multiple different time zones.
In Tajo, there are some time zong settings.
@@ -27,7 +27,7 @@
==================
In Tajo, a table property ``timezone`` allows users to specify a time zone that the table uses for reading or writing.
-When each table row are read or written, ```timestamp``` and ```time``` column values are adjusted by a given time zone if it is set.
+When each table row are read or written, ```timestamp``` column values are adjusted by a given time zone if it is set.
You can specify a table time zone as follows:
diff --git a/tajo-jdbc/src/test/java/org/apache/tajo/jdbc/TestResultSet.java b/tajo-jdbc/src/test/java/org/apache/tajo/jdbc/TestResultSet.java
index 0e9b2a7..c04e3b2 100644
--- a/tajo-jdbc/src/test/java/org/apache/tajo/jdbc/TestResultSet.java
+++ b/tajo-jdbc/src/test/java/org/apache/tajo/jdbc/TestResultSet.java
@@ -23,11 +23,8 @@
import com.google.protobuf.ByteString;
import org.apache.hadoop.fs.Path;
-import org.apache.tajo.IntegrationTest;
-import org.apache.tajo.TajoConstants;
+import org.apache.tajo.*;
import org.apache.tajo.TajoProtos.CodecType;
-import org.apache.tajo.TajoTestingCluster;
-import org.apache.tajo.TpchTestBase;
import org.apache.tajo.catalog.*;
import org.apache.tajo.catalog.statistics.TableStats;
import org.apache.tajo.client.TajoClient;
@@ -50,6 +47,7 @@
import java.sql.*;
import java.util.Calendar;
import java.util.TimeZone;
+import java.util.concurrent.TimeUnit;
import static org.junit.Assert.*;
@@ -73,7 +71,8 @@
.add("deptname", Type.TEXT)
.add("score", Type.INT4)
.build();
- scoreMeta = CatalogUtil.newTableMeta("TEXT");
+
+ scoreMeta = CatalogUtil.newTableMeta(BuiltinStorages.TEXT, conf);
rowBlock = new MemoryRowBlock(SchemaUtil.toDataTypes(scoreSchema));
TableStats stats = new TableStats();
@@ -240,7 +239,13 @@
assertEquals(Timestamp.valueOf("2014-01-01 01:00:00"), timestamp);
// assert with timezone
- Calendar cal = Calendar.getInstance(TimeZone.getTimeZone("GMT+9"));
+
+ //Current timezone + 1 hour
+ TimeZone tz = TimeZone.getDefault();
+ tz.setRawOffset(tz.getRawOffset() + (int) TimeUnit.HOURS.toMillis(1));
+
+ Calendar cal = Calendar.getInstance(tz);
+ assertEquals(tz.getRawOffset(), cal.getTimeZone().getRawOffset());
date = res.getDate(1, cal);
assertNotNull(date);
assertEquals("2014-01-01", date.toString());
@@ -249,21 +254,21 @@
assertNotNull(date);
assertEquals("2014-01-01", date.toString());
- time = res.getTime(2, cal);
+ time = res.getTime(2);
assertNotNull(time);
- assertEquals("10:00:00", time.toString());
+ assertEquals("01:00:00", time.toString());
- time = res.getTime("col2", cal);
+ time = res.getTime("col2");
assertNotNull(time);
- assertEquals("10:00:00", time.toString());
+ assertEquals("01:00:00", time.toString());
timestamp = res.getTimestamp(3, cal);
assertNotNull(timestamp);
- assertEquals("2014-01-01 10:00:00.0", timestamp.toString());
+ assertEquals("2014-01-01 02:00:00.0", timestamp.toString());
timestamp = res.getTimestamp("col3", cal);
assertNotNull(timestamp);
- assertEquals("2014-01-01 10:00:00.0", timestamp.toString());
+ assertEquals("2014-01-01 02:00:00.0", timestamp.toString());
} finally {
if (res != null) {
res.close();
diff --git a/tajo-jdbc/src/test/java/org/apache/tajo/jdbc/TestTajoJdbc.java b/tajo-jdbc/src/test/java/org/apache/tajo/jdbc/TestTajoJdbc.java
index fb1dd61..faaba71 100644
--- a/tajo-jdbc/src/test/java/org/apache/tajo/jdbc/TestTajoJdbc.java
+++ b/tajo-jdbc/src/test/java/org/apache/tajo/jdbc/TestTajoJdbc.java
@@ -681,4 +681,103 @@
}
}
}
+
+ @Test
+ public void testTableValueWithTimeZone() throws Exception {
+ String tableName = CatalogUtil.normalizeIdentifier("testTableValueWithTimeZone");
+
+ Statement stmt = null;
+ ResultSet res = null;
+ Connection conn = null;
+ try {
+ String connUri = buildConnectionUri(tajoMasterAddress.getHostName(), tajoMasterAddress.getPort(),
+ DEFAULT_DATABASE_NAME);
+ connUri = connUri + "?timezone=GMT";
+
+ conn = DriverManager.getConnection(connUri);
+ assertTrue(conn.isValid(100));
+
+ stmt = conn.createStatement();
+ stmt.executeUpdate("create table " + tableName + " (cdate timestamp)");
+ res = stmt.executeQuery("insert overwrite into " + tableName + " select TIMESTAMP '2016-04-01 00:00:00'");
+ cleanupQuery(res);
+
+ res = stmt.executeQuery("select * from " + tableName);
+ assertTrue(res.next());
+ assertEquals("2016-04-01 00:00:00", res.getString(1));
+ cleanupQuery(res);
+ stmt.close();
+ conn.close();
+
+
+ // set time zone ('Asia/Tokyo' offset +9)
+ connUri = buildConnectionUri(tajoMasterAddress.getHostName(), tajoMasterAddress.getPort(),
+ DEFAULT_DATABASE_NAME);
+ connUri = connUri + "?timezone=Asia/Tokyo";
+
+ conn = DriverManager.getConnection(connUri);
+ assertTrue(conn.isValid(100));
+
+ stmt = conn.createStatement();
+
+ res = stmt.executeQuery("select * from " + tableName);
+ assertTrue(res.next());
+ assertEquals("2016-04-01 09:00:00", res.getString(1));
+ } finally {
+ cleanupQuery(res);
+ if (stmt != null) {
+ stmt.close();
+ }
+
+ if(conn != null) {
+ conn.close();
+ }
+ }
+ }
+
+ @Test
+ public void testNonFromQueryWithTimeZone() throws Exception {
+ Statement stmt = null;
+ ResultSet res = null;
+ Connection conn = null;
+ try {
+ String connUri = buildConnectionUri(tajoMasterAddress.getHostName(), tajoMasterAddress.getPort(),
+ DEFAULT_DATABASE_NAME);
+ connUri = connUri + "?timezone=GMT";
+
+ conn = DriverManager.getConnection(connUri);
+ assertTrue(conn.isValid(100));
+
+ stmt = conn.createStatement();
+
+ res = stmt.executeQuery("select TIMESTAMP '2016-04-01 00:00:00'");
+ assertTrue(res.next());
+ assertEquals("2016-04-01 00:00:00", res.getString(1));
+ cleanupQuery(res);
+ stmt.close();
+ conn.close();
+
+ // set different timezone
+ connUri = buildConnectionUri(tajoMasterAddress.getHostName(), tajoMasterAddress.getPort(),
+ DEFAULT_DATABASE_NAME);
+ connUri = connUri + "?timezone=Asia/Tokyo";
+
+ conn = DriverManager.getConnection(connUri);
+ assertTrue(conn.isValid(100));
+
+ stmt = conn.createStatement();
+ res = stmt.executeQuery("select TIMESTAMP '2016-04-01 00:00:00'");
+ assertTrue(res.next());
+ assertEquals("2016-04-01 00:00:00", res.getString(1));
+ } finally {
+ cleanupQuery(res);
+ if (stmt != null) {
+ stmt.close();
+ }
+
+ if(conn != null) {
+ conn.close();
+ }
+ }
+ }
}
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/ExprAnnotator.java b/tajo-plan/src/main/java/org/apache/tajo/plan/ExprAnnotator.java
index e87d33b..d5e9354 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/ExprAnnotator.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/ExprAnnotator.java
@@ -21,9 +21,9 @@
import com.google.common.collect.Sets;
import org.apache.commons.collections.set.UnmodifiableSet;
import org.apache.tajo.OverridableConf;
-import org.apache.tajo.SessionVars;
import org.apache.tajo.algebra.*;
import org.apache.tajo.catalog.*;
+import org.apache.tajo.catalog.CatalogUtil.Direction;
import org.apache.tajo.common.TajoDataTypes;
import org.apache.tajo.datum.*;
import org.apache.tajo.exception.*;
@@ -120,10 +120,13 @@
Type toBeCasted = TUtil.getFromNestedMap(CatalogUtil.OPERATION_CASTING_MAP, lhsType, rhsType);
if (toBeCasted != null) { // if not null, one of either should be converted to another type.
// Overwrite lhs, rhs, or both with cast expression.
- if (lhsType != toBeCasted) {
+
+ Direction direction = CatalogUtil.getCastingDirection(lhsType, rhsType);
+
+ if (lhsType != toBeCasted && (direction == Direction.BOTH || direction == Direction.LHS)) {
lhs = convertType(ctx, lhs, CatalogUtil.newSimpleDataType(toBeCasted));
}
- if (rhsType != toBeCasted) {
+ if (rhsType != toBeCasted && (direction == Direction.BOTH || direction == Direction.RHS)) {
rhs = convertType(ctx, rhs, CatalogUtil.newSimpleDataType(toBeCasted));
}
}
@@ -781,15 +784,9 @@
ConstEval constEval = (ConstEval) child;
// some cast operation may require earlier evaluation with timezone.
- TimeZone tz = null;
- if (ctx.queryContext.containsKey(SessionVars.TIMEZONE)) {
- String tzId = ctx.queryContext.get(SessionVars.TIMEZONE);
- tz = TimeZone.getTimeZone(tzId);
- }
-
return new ConstEval(
DatumFactory.cast(constEval.getValue(),
- LogicalPlanner.convertDataType(expr.getTarget()).getDataType(), tz));
+ LogicalPlanner.convertDataType(expr.getTarget()).getDataType(), ctx.timeZone));
} else {
return new CastEval(ctx.queryContext, child, LogicalPlanner.convertDataType(expr.getTarget()).getDataType());
@@ -858,11 +855,7 @@
TimeMeta tm = new TimeMeta();
DateTimeUtil.toJulianTimeMeta(timestamp, tm);
-
- if (ctx.queryContext.containsKey(SessionVars.TIMEZONE)) {
- TimeZone tz = TimeZone.getTimeZone(ctx.queryContext.get(SessionVars.TIMEZONE));
- DateTimeUtil.toUTCTimezone(tm, tz);
- }
+ DateTimeUtil.toUTCTimezone(tm, ctx.timeZone);
return new ConstEval(new TimestampDatum(DateTimeUtil.toJulianTimestamp(tm)));
}
@@ -889,11 +882,6 @@
TimeDatum timeDatum = new TimeDatum(time);
TimeMeta tm = timeDatum.asTimeMeta();
- if (ctx.queryContext.containsKey(SessionVars.TIMEZONE)) {
- TimeZone tz = TimeZone.getTimeZone(ctx.queryContext.get(SessionVars.TIMEZONE));
- DateTimeUtil.toUTCTimezone(tm, tz);
- }
-
return new ConstEval(new TimeDatum(DateTimeUtil.toTime(tm)));
}
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java
index f74e16f..5336906 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java
@@ -103,10 +103,8 @@
this.evalOptimizer = evalOptimizer;
// session's time zone
- if (context.containsKey(SessionVars.TIMEZONE)) {
- String timezoneId = context.get(SessionVars.TIMEZONE);
- timeZone = TimeZone.getTimeZone(timezoneId);
- }
+ String timezoneId = context.get(SessionVars.TIMEZONE);
+ this.timeZone = TimeZone.getTimeZone(timezoneId);
this.debugOrUnitTests = debugOrUnitTests;
}
@@ -1899,7 +1897,8 @@
}
// Set default storage properties to table
- createTableNode.setOptions(CatalogUtil.newDefaultProperty(createTableNode.getStorageType()));
+ createTableNode.setOptions(
+ CatalogUtil.newDefaultProperty(createTableNode.getStorageType(), context.getQueryContext().getConf()));
// Priority to apply table properties
// 1. Explicit table properties specified in WITH clause
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/TablePropertyUtil.java b/tajo-plan/src/main/java/org/apache/tajo/plan/TablePropertyUtil.java
index 25bd5b4..71e82d4 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/TablePropertyUtil.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/TablePropertyUtil.java
@@ -21,6 +21,7 @@
import org.apache.tajo.OverridableConf;
import org.apache.tajo.SessionVars;
import org.apache.tajo.catalog.TableMeta;
+import org.apache.tajo.conf.TajoConf;
import org.apache.tajo.plan.logical.CreateTableNode;
import org.apache.tajo.plan.logical.ScanNode;
import org.apache.tajo.storage.StorageConstants;
@@ -65,8 +66,8 @@
public static void setTableProperty(OverridableConf context, ScanNode node) {
TableMeta meta = node.getTableDesc().getMeta();
- setProperty(context, SessionVars.TIMEZONE, meta, StorageConstants.TIMEZONE);
- setProperty(context, SessionVars.NULL_CHAR, meta, StorageConstants.TEXT_NULL);
+ // set default time zone, if there is no table timezone
+ setProperty(context.getConf(), TajoConf.ConfVars.$TIMEZONE, meta, StorageConstants.TIMEZONE);
}
/**
@@ -86,4 +87,20 @@
meta.putProperty(propertyKey, context.get(sessionVarKey));
}
}
+
+ /**
+ * If there is no table property for the propertyKey, set default property from system conf to the table.
+ *
+ * @param conf TajoConf
+ * @param confVarKey system variable key
+ * @param meta TableMeta
+ * @param propertyKey table property key
+ */
+ private static void setProperty(TajoConf conf, TajoConf.ConfVars confVarKey,
+ TableMeta meta, String propertyKey) {
+
+ if (!meta.containsProperty(propertyKey)) {
+ meta.putProperty(propertyKey, conf.getVar(confVarKey));
+ }
+ }
}
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/expr/CastEval.java b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/CastEval.java
index 653eeb0..975b69e 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/expr/CastEval.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/CastEval.java
@@ -20,13 +20,13 @@
import com.google.gson.annotations.Expose;
import org.apache.tajo.OverridableConf;
-import org.apache.tajo.SessionVars;
+import org.apache.tajo.annotation.Nullable;
+import org.apache.tajo.catalog.Schema;
import org.apache.tajo.catalog.TypeConverter;
import org.apache.tajo.datum.Datum;
import org.apache.tajo.datum.DatumFactory;
import org.apache.tajo.storage.Tuple;
import org.apache.tajo.type.Type;
-import org.apache.tajo.util.TUtil;
import java.util.TimeZone;
@@ -34,16 +34,11 @@
public class CastEval extends UnaryEval implements Cloneable {
@Expose private DataType target;
- @Expose private TimeZone timezone;
+ private TimeZone timezone;
public CastEval(OverridableConf context, EvalNode operand, DataType target) {
super(EvalType.CAST, operand);
this.target = target;
-
- if (context.containsKey(SessionVars.TIMEZONE)) {
- String timezoneId = context.get(SessionVars.TIMEZONE);
- timezone = TimeZone.getTimeZone(timezoneId);
- }
}
public EvalNode getOperand() {
@@ -55,20 +50,20 @@
return TypeConverter.convert(target);
}
- public boolean hasTimeZone() {
- return this.timezone != null;
- }
-
- public TimeZone getTimezone() {
- return this.timezone;
- }
-
@Override
public String getName() {
return target.getType().name();
}
@Override
+ public EvalNode bind(@Nullable EvalContext evalContext, Schema schema) {
+ if (evalContext != null) {
+ timezone = evalContext.getTimeZone();
+ }
+ return super.bind(evalContext, schema);
+ }
+
+ @Override
@SuppressWarnings("unchecked")
public Datum eval(Tuple tuple) {
super.eval(tuple);
@@ -89,7 +84,6 @@
final int prime = 31;
int result = super.hashCode();
result = prime * result + ((target == null) ? 0 : target.hashCode());
- result = prime * result + ((timezone == null) ? 0 : timezone.hashCode());
return result;
}
@@ -100,8 +94,7 @@
CastEval another = (CastEval) obj;
boolean b1 = child.equals(another.child);
boolean b2 = target.equals(another.target);
- boolean b3 = TUtil.checkEquals(timezone, another.timezone);
- return b1 && b2 && b3;
+ return b1 && b2;
} else {
return false;
}
@@ -113,9 +106,7 @@
if (target != null) {
clone.target = target;
}
- if (timezone != null) {
- clone.timezone = timezone;
- }
+
return clone;
}
}
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/expr/EvalContext.java b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/EvalContext.java
index e6099bc..9d287d4 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/expr/EvalContext.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/EvalContext.java
@@ -23,9 +23,11 @@
import java.util.Collection;
import java.util.HashMap;
import java.util.Map;
+import java.util.TimeZone;
public class EvalContext {
private final Map<EvalNode, TajoScriptEngine> scriptEngineMap = new HashMap<>();
+ private TimeZone timeZone;
public void addScriptEngine(EvalNode evalNode, TajoScriptEngine scriptExecutor) {
this.scriptEngineMap.put(evalNode, scriptExecutor);
@@ -43,4 +45,16 @@
public Collection<TajoScriptEngine> getAllScriptEngines() {
return this.scriptEngineMap.values();
}
+
+ public TimeZone getTimeZone() {
+ return timeZone;
+ }
+
+ public void setTimeZone(TimeZone timeZone) {
+ this.timeZone = timeZone;
+ }
+
+ public boolean hasTimeZone() {
+ return timeZone != null;
+ }
}
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/expr/EvalNode.java b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/EvalNode.java
index 665a770..8a501c0 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/expr/EvalNode.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/EvalNode.java
@@ -22,7 +22,6 @@
import org.apache.tajo.annotation.Nullable;
import org.apache.tajo.catalog.Schema;
import org.apache.tajo.common.ProtoObject;
-import org.apache.tajo.common.TajoDataTypes.DataType;
import org.apache.tajo.datum.Datum;
import org.apache.tajo.json.GsonObject;
import org.apache.tajo.plan.serder.EvalNodeSerializer;
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/expr/GeneralFunctionEval.java b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/GeneralFunctionEval.java
index febadc0..3f70397 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/expr/GeneralFunctionEval.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/GeneralFunctionEval.java
@@ -44,8 +44,14 @@
super.bind(evalContext, schema);
try {
this.funcInvoke = FunctionInvoke.newInstance(funcDesc);
- if (evalContext != null && evalContext.hasScriptEngine(this)) {
- this.invokeContext.setScriptEngine(evalContext.getScriptEngine(this));
+ if (evalContext != null) {
+ if (evalContext.hasScriptEngine(this)) {
+ this.invokeContext.setScriptEngine(evalContext.getScriptEngine(this));
+ }
+
+ if (evalContext.hasTimeZone()) {
+ this.invokeContext.setTimeZone(evalContext.getTimeZone());
+ }
}
this.funcInvoke.init(invokeContext);
} catch (IOException e) {
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/exprrewrite/rules/ConstantFolding.java b/tajo-plan/src/main/java/org/apache/tajo/plan/exprrewrite/rules/ConstantFolding.java
index 96186d3..e026fa5 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/exprrewrite/rules/ConstantFolding.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/exprrewrite/rules/ConstantFolding.java
@@ -18,6 +18,7 @@
package org.apache.tajo.plan.exprrewrite.rules;
+import org.apache.tajo.SessionVars;
import org.apache.tajo.datum.Datum;
import org.apache.tajo.plan.LogicalPlanner;
import org.apache.tajo.plan.annotator.Prioritized;
@@ -27,10 +28,7 @@
import org.apache.tajo.plan.function.python.TajoScriptEngine;
import java.io.IOException;
-import java.util.Arrays;
-import java.util.HashSet;
-import java.util.Set;
-import java.util.Stack;
+import java.util.*;
@Prioritized(priority = 10)
public class ConstantFolding extends SimpleEvalNodeVisitor<LogicalPlanner.PlanContext>
@@ -71,7 +69,12 @@
unaryEval.setChild(child);
if (child.getType() == EvalType.CONST) {
- unaryEval.bind(null, null);
+
+ // session's time zone
+ String timezoneId = context.getQueryContext().get(SessionVars.TIMEZONE);
+ EvalContext evalContext = new EvalContext();
+ evalContext.setTimeZone(TimeZone.getTimeZone(timezoneId));
+ unaryEval.bind(evalContext, null);
return new ConstEval(unaryEval.eval(null));
}
@@ -95,11 +98,15 @@
}
if (constantOfAllDescendents && evalNode.getType() == EvalType.FUNCTION) {
+ EvalContext evalContext = new EvalContext();
+ // session's time zone
+ String timezoneId = context.getQueryContext().get(SessionVars.TIMEZONE);
+ evalContext.setTimeZone(TimeZone.getTimeZone(timezoneId));
+
if (evalNode.getFuncDesc().getInvocation().hasPython()) {
TajoScriptEngine executor = new PythonScriptEngine(evalNode.getFuncDesc());
try {
executor.start(context.getQueryContext().getConf());
- EvalContext evalContext = new EvalContext();
evalContext.addScriptEngine(evalNode, executor);
evalNode.bind(evalContext, null);
Datum funcRes = evalNode.eval(null);
@@ -109,7 +116,7 @@
throw new RuntimeException(e);
}
} else {
- evalNode.bind(null, null);
+ evalNode.bind(evalContext, null);
return new ConstEval(evalNode.eval(null));
}
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/function/ClassBasedScalarFunctionInvoke.java b/tajo-plan/src/main/java/org/apache/tajo/plan/function/ClassBasedScalarFunctionInvoke.java
index fcdb261..3b9a853 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/function/ClassBasedScalarFunctionInvoke.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/function/ClassBasedScalarFunctionInvoke.java
@@ -47,6 +47,7 @@
@Override
public void init(FunctionInvokeContext context) {
+ function.setTimeZone(context.getTimeZone());
function.init(context.getQueryContext(), context.getParamTypes());
}
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/function/FunctionInvokeContext.java b/tajo-plan/src/main/java/org/apache/tajo/plan/function/FunctionInvokeContext.java
index fb8dcea..5f051b7 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/function/FunctionInvokeContext.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/function/FunctionInvokeContext.java
@@ -26,6 +26,7 @@
import org.apache.tajo.util.TUtil;
import java.util.Arrays;
+import java.util.TimeZone;
/**
* This class contains some metadata need to execute functions.
@@ -34,6 +35,7 @@
private OverridableConf queryContext;
private FunctionEval.ParamType[] paramTypes;
private TajoScriptEngine scriptEngine;
+ private TimeZone timeZone;
public FunctionInvokeContext(@Nullable OverridableConf queryContext, FunctionEval.ParamType[] paramTypes) {
this.queryContext = queryContext;
@@ -60,6 +62,14 @@
return scriptEngine;
}
+ public TimeZone getTimeZone() {
+ return timeZone;
+ }
+
+ public void setTimeZone(TimeZone timeZone) {
+ this.timeZone = timeZone;
+ }
+
@Override
public int hashCode() {
return Objects.hashCode(queryContext, Arrays.hashCode(paramTypes));
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/function/GeneralFunction.java b/tajo-plan/src/main/java/org/apache/tajo/plan/function/GeneralFunction.java
index 39db5c6..9f20489 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/function/GeneralFunction.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/function/GeneralFunction.java
@@ -18,6 +18,7 @@
package org.apache.tajo.plan.function;
+import com.google.gson.annotations.Expose;
import org.apache.tajo.OverridableConf;
import org.apache.tajo.catalog.Column;
import org.apache.tajo.catalog.json.CatalogGsonHelper;
@@ -28,8 +29,12 @@
import org.apache.tajo.plan.expr.FunctionEval;
import org.apache.tajo.storage.Tuple;
+import java.util.TimeZone;
+
@Deprecated
public abstract class GeneralFunction extends Function implements GsonObject {
+ @Expose
+ protected TimeZone timeZone;
public GeneralFunction(Column[] definedArgs) {
super(definedArgs);
}
@@ -56,4 +61,16 @@
public CatalogProtos.FunctionType getFunctionType() {
return CatalogProtos.FunctionType.GENERAL;
}
+
+ public void setTimeZone(TimeZone timeZone) {
+ this.timeZone = timeZone;
+ }
+
+ public TimeZone getTimeZone() {
+ return timeZone;
+ }
+
+ public boolean hasTimeZone() {
+ return timeZone != null;
+ }
}
\ No newline at end of file
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/function/python/PythonScriptEngine.java b/tajo-plan/src/main/java/org/apache/tajo/plan/function/python/PythonScriptEngine.java
index 2e2e7b5..dcaac0c 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/function/python/PythonScriptEngine.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/function/python/PythonScriptEngine.java
@@ -23,6 +23,7 @@
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
+import org.apache.tajo.BuiltinStorages;
import org.apache.tajo.TajoConstants;
import org.apache.tajo.catalog.*;
import org.apache.tajo.catalog.proto.CatalogProtos;
@@ -40,6 +41,7 @@
import org.apache.tajo.storage.VTuple;
import org.apache.tajo.unit.StorageUnit;
import org.apache.tajo.util.FileUtil;
+import org.apache.tajo.util.TUtil;
import java.io.*;
import java.net.URI;
@@ -269,7 +271,7 @@
FUNCTION_TYPE,
}
- private Configuration systemConf;
+ private TajoConf systemConf;
private Process process; // Handle to the external execution of python functions
@@ -287,7 +289,7 @@
private int[] projectionCols;
private final CSVLineSerDe lineSerDe = new CSVLineSerDe();
- private final TableMeta pipeMeta = CatalogUtil.newTableMeta("TEXT");
+ private TableMeta pipeMeta;
private final Tuple EMPTY_INPUT = new VTuple(0);
private final Schema EMPTY_SCHEMA = SchemaBuilder.builder().build();
@@ -313,8 +315,9 @@
}
@Override
- public void start(Configuration systemConf) throws IOException {
- this.systemConf = systemConf;
+ public void start(Configuration conf) throws IOException {
+ this.systemConf = TUtil.checkTypeAndGet(conf, TajoConf.class);
+ this.pipeMeta = CatalogUtil.newTableMeta(BuiltinStorages.TEXT, systemConf);
startUdfController();
setStreams();
createInputHandlers();
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/function/stream/TextFieldSerializerDeserializer.java b/tajo-plan/src/main/java/org/apache/tajo/plan/function/stream/TextFieldSerializerDeserializer.java
index 7d03f47..9638634 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/function/stream/TextFieldSerializerDeserializer.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/function/stream/TextFieldSerializerDeserializer.java
@@ -22,16 +22,17 @@
import io.netty.buffer.ByteBuf;
import io.netty.util.CharsetUtil;
import org.apache.commons.codec.binary.Base64;
-import org.apache.tajo.TajoConstants;
import org.apache.tajo.catalog.CatalogUtil;
import org.apache.tajo.catalog.TableMeta;
import org.apache.tajo.common.TajoDataTypes;
+import org.apache.tajo.conf.TajoConf;
import org.apache.tajo.datum.*;
import org.apache.tajo.datum.protobuf.ProtobufJsonFormat;
import org.apache.tajo.exception.TajoRuntimeException;
import org.apache.tajo.exception.UnsupportedException;
import org.apache.tajo.exception.ValueTooLongForTypeCharactersException;
import org.apache.tajo.storage.StorageConstants;
+import org.apache.tajo.util.Bytes;
import org.apache.tajo.util.NumberUtil;
import java.io.IOException;
@@ -45,12 +46,12 @@
private static ProtobufJsonFormat protobufJsonFormat = ProtobufJsonFormat.getInstance();
private final CharsetDecoder decoder = CharsetUtil.getDecoder(CharsetUtil.UTF_8);
- private final boolean hasTimezone;
- private final TimeZone timezone;
+ private static final TajoConf TAJO_CONF = new TajoConf();
+ private final TimeZone tableTimezone;
public TextFieldSerializerDeserializer(TableMeta meta) {
- hasTimezone = meta.containsProperty(StorageConstants.TIMEZONE);
- timezone = TimeZone.getTimeZone(meta.getProperty(StorageConstants.TIMEZONE, TajoConstants.DEFAULT_SYSTEM_TIMEZONE));
+ tableTimezone = TimeZone.getTimeZone(
+ meta.getProperty(StorageConstants.TIMEZONE, TAJO_CONF.getSystemTimezone().getID()));
}
private static boolean isNull(ByteBuf val, ByteBuf nullBytes) {
@@ -112,20 +113,13 @@
out.write(bytes);
break;
case TIME:
- if (hasTimezone) {
- bytes = ((TimeDatum) datum).toString(timezone, true).getBytes();
- } else {
- bytes = datum.asTextBytes();
- }
+ bytes = datum.asTextBytes();
length = bytes.length;
out.write(bytes);
break;
case TIMESTAMP:
- if (hasTimezone) {
- bytes = ((TimestampDatum) datum).toString(timezone, true).getBytes();
- } else {
- bytes = datum.asTextBytes();
- }
+ // UTC to table timezone
+ bytes = ((TimestampDatum) datum).toString(tableTimezone, true).getBytes(Bytes.UTF8_CHARSET);
length = bytes.length;
out.write(bytes);
break;
@@ -209,22 +203,13 @@
decoder.decode(buf.nioBuffer(buf.readerIndex(), buf.readableBytes())).toString());
break;
case TIME:
- if (hasTimezone) {
- datum = DatumFactory.createTime(
- decoder.decode(buf.nioBuffer(buf.readerIndex(), buf.readableBytes())).toString(), timezone);
- } else {
- datum = DatumFactory.createTime(
- decoder.decode(buf.nioBuffer(buf.readerIndex(), buf.readableBytes())).toString());
- }
+ datum = DatumFactory.createTime(
+ decoder.decode(buf.nioBuffer(buf.readerIndex(), buf.readableBytes())).toString());
break;
case TIMESTAMP:
- if (hasTimezone) {
- datum = DatumFactory.createTimestamp(
- decoder.decode(buf.nioBuffer(buf.readerIndex(), buf.readableBytes())).toString(), timezone);
- } else {
- datum = DatumFactory.createTimestamp(
- decoder.decode(buf.nioBuffer(buf.readerIndex(), buf.readableBytes())).toString());
- }
+ // table timezone to UTC
+ datum = DatumFactory.createTimestamp(
+ decoder.decode(buf.nioBuffer(buf.readerIndex(), buf.readableBytes())).toString(), tableTimezone);
break;
case INTERVAL:
datum = DatumFactory.createInterval(
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/serder/EvalNodeSerializer.java b/tajo-plan/src/main/java/org/apache/tajo/plan/serder/EvalNodeSerializer.java
index 759867f..db5215b 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/serder/EvalNodeSerializer.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/serder/EvalNodeSerializer.java
@@ -120,9 +120,6 @@
} else if (unary.getType() == EvalType.CAST) {
CastEval castEval = (CastEval) unary;
unaryBuilder.setCastingType(TypeConverter.convert(castEval.getValueType()));
- if (castEval.hasTimeZone()) {
- unaryBuilder.setTimezone(castEval.getTimezone().getID());
- }
}
// registering itself and building EvalNode
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/ExprsVerifier.java b/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/ExprsVerifier.java
index 2040ae3..2afef4a 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/ExprsVerifier.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/ExprsVerifier.java
@@ -163,7 +163,8 @@
}
if (leftType == TIME &&
- (rightType == DATE || rightType == INTERVAL || rightType == TIME)) {
+ (rightType == DATE || rightType == INTERVAL || rightType == TIME ||
+ rightType == TIMESTAMP)) {
return;
}
diff --git a/tajo-plan/src/main/proto/Plan.proto b/tajo-plan/src/main/proto/Plan.proto
index 0cd0c32..c50429f 100644
--- a/tajo-plan/src/main/proto/Plan.proto
+++ b/tajo-plan/src/main/proto/Plan.proto
@@ -431,7 +431,6 @@
required int32 child_id = 1;
optional DataType castingType = 2;
optional bool negative = 3;
- optional string timezone = 4;
}
message BinaryEval {
diff --git a/tajo-plan/src/test/java/org/apache/tajo/plan/TestLogicalNode.java b/tajo-plan/src/test/java/org/apache/tajo/plan/TestLogicalNode.java
index 3e44e3b..4ebb796 100644
--- a/tajo-plan/src/test/java/org/apache/tajo/plan/TestLogicalNode.java
+++ b/tajo-plan/src/test/java/org/apache/tajo/plan/TestLogicalNode.java
@@ -19,11 +19,13 @@
package org.apache.tajo.plan;
import org.apache.hadoop.fs.Path;
+import org.apache.tajo.BuiltinStorages;
import org.apache.tajo.catalog.CatalogUtil;
import org.apache.tajo.catalog.Column;
import org.apache.tajo.catalog.Schema;
import org.apache.tajo.catalog.SchemaBuilder;
import org.apache.tajo.common.TajoDataTypes.Type;
+import org.apache.tajo.conf.TajoConf;
import org.apache.tajo.plan.logical.GroupbyNode;
import org.apache.tajo.plan.logical.JoinNode;
import org.apache.tajo.plan.logical.ScanNode;
@@ -36,6 +38,8 @@
@Test
public void testEquals() {
+ TajoConf conf = new TajoConf();
+
Schema schema = SchemaBuilder.builder()
.add("id", Type.INT4)
.add("name", Type.TEXT)
@@ -44,13 +48,15 @@
GroupbyNode groupbyNode = new GroupbyNode(0);
groupbyNode.setGroupingColumns(new Column[]{schema.getColumn(1), schema.getColumn(2)});
ScanNode scanNode = new ScanNode(0);
- scanNode.init(CatalogUtil.newTableDesc("in", schema, CatalogUtil.newTableMeta("TEXT"), new Path("in")));
+ scanNode.init(CatalogUtil.newTableDesc("in", schema,
+ CatalogUtil.newTableMeta(BuiltinStorages.TEXT, conf), new Path("in")));
GroupbyNode groupbyNode2 = new GroupbyNode(0);
groupbyNode2.setGroupingColumns(new Column[]{schema.getColumn(1), schema.getColumn(2)});
JoinNode joinNode = new JoinNode(0);
ScanNode scanNode2 = new ScanNode(0);
- scanNode2.init(CatalogUtil.newTableDesc("in2", schema, CatalogUtil.newTableMeta("TEXT"), new Path("in2")));
+ scanNode2.init(CatalogUtil.newTableDesc("in2", schema,
+ CatalogUtil.newTableMeta(BuiltinStorages.TEXT, conf), new Path("in2")));
groupbyNode.setChild(scanNode);
groupbyNode2.setChild(joinNode);
@@ -61,7 +67,8 @@
assertFalse(groupbyNode.deepEquals(groupbyNode2));
ScanNode scanNode3 = new ScanNode(0);
- scanNode3.init(CatalogUtil.newTableDesc("in", schema, CatalogUtil.newTableMeta("TEXT"), new Path("in")));
+ scanNode3.init(CatalogUtil.newTableDesc("in", schema,
+ CatalogUtil.newTableMeta(BuiltinStorages.TEXT, conf), new Path("in")));
groupbyNode2.setChild(scanNode3);
assertTrue(groupbyNode.equals(groupbyNode2));
diff --git a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/StorageUtil.java b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/StorageUtil.java
index 442519b..263fb05 100644
--- a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/StorageUtil.java
+++ b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/StorageUtil.java
@@ -21,6 +21,7 @@
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.tajo.conf.TajoConf;
import sun.nio.ch.DirectBuffer;
import java.io.DataInput;
@@ -31,6 +32,8 @@
public class StorageUtil extends StorageConstants {
+ public static final TajoConf TAJO_CONF = new TajoConf();
+
public static Path concatPath(String parent, String...childs) {
return concatPath(new Path(parent), childs);
}
diff --git a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/TextSerializerDeserializer.java b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/TextSerializerDeserializer.java
index 1ec13bc..0bc2168 100644
--- a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/TextSerializerDeserializer.java
+++ b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/TextSerializerDeserializer.java
@@ -31,7 +31,6 @@
import java.io.IOException;
import java.io.OutputStream;
-import java.util.TimeZone;
// Compatibility with Apache Hive
@Deprecated
@@ -101,12 +100,7 @@
out.write(bytes);
break;
case TIME:
- bytes = TimeDatum.asChars(tuple.getTimeDate(index), TimeZone.getDefault(), true).getBytes();
- length = bytes.length;
- out.write(bytes);
- break;
- case TIMESTAMP:
- bytes = TimestampDatum.asChars(tuple.getTimeDate(index), TimeZone.getDefault(), true).getBytes();
+ bytes = tuple.getTextBytes(index);
length = bytes.length;
out.write(bytes);
break;
@@ -185,10 +179,6 @@
datum = isNull(bytes, offset, length, nullCharacters) ? NullDatum.get()
: DatumFactory.createTime(new String(bytes, offset, length));
break;
- case TIMESTAMP:
- datum = isNull(bytes, offset, length, nullCharacters) ? NullDatum.get()
- : DatumFactory.createTimestamp(new String(bytes, offset, length));
- break;
case INTERVAL:
datum = isNull(bytes, offset, length, nullCharacters) ? NullDatum.get()
: DatumFactory.createInterval(new String(bytes, offset, length));
diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/json/JsonLineDeserializer.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/json/JsonLineDeserializer.java
index 335f12b..5e35239 100644
--- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/json/JsonLineDeserializer.java
+++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/json/JsonLineDeserializer.java
@@ -26,7 +26,6 @@
import net.minidev.json.parser.JSONParser;
import net.minidev.json.parser.ParseException;
import org.apache.commons.net.util.Base64;
-import org.apache.tajo.TajoConstants;
import org.apache.tajo.catalog.*;
import org.apache.tajo.common.TajoDataTypes.Type;
import org.apache.tajo.datum.DatumFactory;
@@ -34,6 +33,7 @@
import org.apache.tajo.exception.NotImplementedException;
import org.apache.tajo.exception.TajoRuntimeException;
import org.apache.tajo.storage.StorageConstants;
+import org.apache.tajo.storage.StorageUtil;
import org.apache.tajo.storage.Tuple;
import org.apache.tajo.storage.text.TextLineDeserializer;
import org.apache.tajo.storage.text.TextLineParsingError;
@@ -51,7 +51,6 @@
private final String [] projectedPaths;
private final CharsetDecoder decoder = CharsetUtil.getDecoder(CharsetUtil.UTF_8);
- private final boolean hasTimezone;
private final TimeZone timezone;
public JsonLineDeserializer(Schema schema, TableMeta meta, Column [] projected) {
@@ -60,8 +59,8 @@
projectedPaths = SchemaUtil.convertColumnsToPaths(Lists.newArrayList(projected), true);
types = SchemaUtil.buildTypeMap(schema.getAllColumns(), projectedPaths);
- hasTimezone = meta.containsProperty(StorageConstants.TIMEZONE);
- timezone = TimeZone.getTimeZone(meta.getProperty(StorageConstants.TIMEZONE, TajoConstants.DEFAULT_SYSTEM_TIMEZONE));
+ timezone = TimeZone.getTimeZone(meta.getProperty(StorageConstants.TIMEZONE,
+ StorageUtil.TAJO_CONF.getSystemTimezone().getID()));
}
@Override
@@ -160,11 +159,7 @@
case TIMESTAMP:
String timestampStr = object.getAsString(fieldName);
if (timestampStr != null) {
- if (hasTimezone) {
- output.put(fieldIndex, DatumFactory.createTimestamp(timestampStr, timezone));
- } else {
- output.put(fieldIndex, DatumFactory.createTimestamp(timestampStr));
- }
+ output.put(fieldIndex, DatumFactory.createTimestamp(timestampStr, timezone));
} else {
output.put(fieldIndex, NullDatum.get());
}
@@ -172,11 +167,7 @@
case TIME:
String timeStr = object.getAsString(fieldName);
if (timeStr != null) {
- if (hasTimezone) {
- output.put(fieldIndex, DatumFactory.createTime(timeStr, timezone));
- } else {
- output.put(fieldIndex, DatumFactory.createTime(timeStr));
- }
+ output.put(fieldIndex, DatumFactory.createTime(timeStr));
} else {
output.put(fieldIndex, NullDatum.get());
}
diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/json/JsonLineSerializer.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/json/JsonLineSerializer.java
index 1885c80..9d14573 100644
--- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/json/JsonLineSerializer.java
+++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/json/JsonLineSerializer.java
@@ -21,18 +21,17 @@
import net.minidev.json.JSONObject;
import org.apache.commons.net.util.Base64;
-import org.apache.tajo.TajoConstants;
import org.apache.tajo.catalog.NestedPathUtil;
import org.apache.tajo.catalog.Schema;
import org.apache.tajo.catalog.SchemaUtil;
import org.apache.tajo.catalog.TableMeta;
import org.apache.tajo.common.TajoDataTypes.Type;
import org.apache.tajo.datum.TextDatum;
-import org.apache.tajo.datum.TimeDatum;
import org.apache.tajo.datum.TimestampDatum;
import org.apache.tajo.exception.NotImplementedException;
import org.apache.tajo.exception.TajoRuntimeException;
import org.apache.tajo.storage.StorageConstants;
+import org.apache.tajo.storage.StorageUtil;
import org.apache.tajo.storage.Tuple;
import org.apache.tajo.storage.text.TextLineSerializer;
@@ -46,7 +45,6 @@
private final Map<String, Type> types;
private final String [] projectedPaths;
- private final boolean hasTimezone;
private final TimeZone timezone;
public JsonLineSerializer(Schema schema, TableMeta meta) {
@@ -55,8 +53,8 @@
projectedPaths = SchemaUtil.convertColumnsToPaths(schema.getAllColumns(), true);
types = SchemaUtil.buildTypeMap(schema.getAllColumns(), projectedPaths);
- hasTimezone = meta.containsProperty(StorageConstants.TIMEZONE);
- timezone = TimeZone.getTimeZone(meta.getProperty(StorageConstants.TIMEZONE, TajoConstants.DEFAULT_SYSTEM_TIMEZONE));
+ timezone = TimeZone.getTimeZone(meta.getProperty(StorageConstants.TIMEZONE,
+ StorageUtil.TAJO_CONF.getSystemTimezone().getID()));
}
@Override
@@ -116,18 +114,10 @@
break;
case TIMESTAMP:
- if (hasTimezone) {
- json.put(fieldName, TimestampDatum.asChars(input.getTimeDate(fieldIndex), timezone, false));
- } else {
- json.put(fieldName, input.asDatum(fieldIndex).asChars());
- }
+ json.put(fieldName, TimestampDatum.asChars(input.getTimeDate(fieldIndex), timezone, false));
break;
case TIME:
- if (hasTimezone) {
- json.put(fieldName, TimeDatum.asChars(input.getTimeDate(fieldIndex), timezone, false));
- } else {
- json.put(fieldName, input.asDatum(fieldIndex).asChars());
- }
+ json.put(fieldName, input.asDatum(fieldIndex).asChars());
break;
case BIT:
diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/orc/ORCAppender.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/orc/ORCAppender.java
index b27c640..d8f320f 100644
--- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/orc/ORCAppender.java
+++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/orc/ORCAppender.java
@@ -23,15 +23,11 @@
import org.apache.orc.CompressionKind;
import org.apache.orc.OrcConf;
import org.apache.orc.TypeDescription;
-import org.apache.tajo.TajoConstants;
import org.apache.tajo.TaskAttemptId;
import org.apache.tajo.catalog.Schema;
import org.apache.tajo.catalog.TableMeta;
import org.apache.tajo.catalog.statistics.TableStats;
-import org.apache.tajo.storage.FileAppender;
-import org.apache.tajo.storage.StorageConstants;
-import org.apache.tajo.storage.TableStatistics;
-import org.apache.tajo.storage.Tuple;
+import org.apache.tajo.storage.*;
import org.apache.tajo.storage.thirdparty.orc.OrcFile;
import org.apache.tajo.storage.thirdparty.orc.OrcFile.EncodingStrategy;
import org.apache.tajo.storage.thirdparty.orc.OrcUtils;
@@ -49,14 +45,13 @@
public ORCAppender(Configuration conf, TaskAttemptId taskAttemptId, Schema schema,
TableMeta meta, Path workDir) {
super(conf, taskAttemptId, schema, meta, workDir);
-
- timezone = meta.containsProperty(StorageConstants.TIMEZONE) ?
- TimeZone.getTimeZone(meta.getProperty(StorageConstants.TIMEZONE)) :
- TimeZone.getDefault();
}
@Override
public void init() throws IOException {
+ timezone = TimeZone.getTimeZone(meta.getProperty(StorageConstants.TIMEZONE,
+ StorageUtil.TAJO_CONF.getSystemTimezone().getID()));
+
writer = OrcFile.createWriter(path, buildWriterOptions(conf, meta, schema), timezone);
if (tableStatsEnabled) {
diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/orc/OrcScanner.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/orc/OrcScanner.java
index c8aa67b..1329955 100644
--- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/orc/OrcScanner.java
+++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/orc/OrcScanner.java
@@ -31,12 +31,12 @@
import org.apache.orc.Reader.Options;
import org.apache.orc.impl.BufferChunk;
import org.apache.orc.impl.InStream;
-import org.apache.tajo.TajoConstants;
import org.apache.tajo.catalog.Schema;
import org.apache.tajo.catalog.TableMeta;
import org.apache.tajo.plan.expr.EvalNode;
import org.apache.tajo.storage.FileScanner;
import org.apache.tajo.storage.StorageConstants;
+import org.apache.tajo.storage.StorageUtil;
import org.apache.tajo.storage.Tuple;
import org.apache.tajo.storage.fragment.Fragment;
import org.apache.tajo.storage.thirdparty.orc.OrcRecordReader;
@@ -67,6 +67,7 @@
private List<StripeInformation> stripes;
protected int rowIndexStride;
private long contentLength, numberOfRows;
+ private TimeZone timeZone;
private List<Integer> versionList;
@@ -236,7 +237,7 @@
public OrcRecordReader createRecordReader() throws IOException {
return new OrcRecordReader(this.stripes, fileSystem, schema, targets, fragment, types, codec, bufferSize,
rowIndexStride, buildReaderOptions(meta), conf,
- TimeZone.getTimeZone(meta.getProperty(StorageConstants.TIMEZONE, TajoConstants.DEFAULT_SYSTEM_TIMEZONE)));
+ timeZone);
}
private static Options buildReaderOptions(TableMeta meta) {
@@ -272,7 +273,9 @@
this.versionList = footerMetaData.versionList;
this.stripes = convertProtoStripesToStripes(rInfo.footer.getStripesList());
- recordReader = createRecordReader();
+ this.timeZone = TimeZone.getTimeZone(meta.getProperty(StorageConstants.TIMEZONE,
+ StorageUtil.TAJO_CONF.getSystemTimezone().getID()));
+ this.recordReader = createRecordReader();
super.init();
}
diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/text/TextFieldSerializerDeserializer.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/text/TextFieldSerializerDeserializer.java
index 53b6bcd..f1600c0 100644
--- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/text/TextFieldSerializerDeserializer.java
+++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/text/TextFieldSerializerDeserializer.java
@@ -20,9 +20,9 @@
import com.google.protobuf.Message;
import io.netty.buffer.ByteBuf;
+import io.netty.buffer.ByteBufUtil;
import io.netty.util.CharsetUtil;
import org.apache.commons.codec.binary.Base64;
-import org.apache.tajo.TajoConstants;
import org.apache.tajo.catalog.Column;
import org.apache.tajo.catalog.Schema;
import org.apache.tajo.catalog.TableMeta;
@@ -32,6 +32,7 @@
import org.apache.tajo.exception.ValueTooLongForTypeCharactersException;
import org.apache.tajo.storage.FieldSerializerDeserializer;
import org.apache.tajo.storage.StorageConstants;
+import org.apache.tajo.storage.StorageUtil;
import org.apache.tajo.storage.Tuple;
import org.apache.tajo.util.Bytes;
import org.apache.tajo.util.NumberUtil;
@@ -47,14 +48,13 @@
private static ProtobufJsonFormat protobufJsonFormat = ProtobufJsonFormat.getInstance();
private final CharsetDecoder decoder = CharsetUtil.getDecoder(CharsetUtil.UTF_8);
- private final boolean hasTimezone;
- private final TimeZone timezone;
+ private final TimeZone tableTimezone;
private Schema schema;
public TextFieldSerializerDeserializer(TableMeta meta) {
- hasTimezone = meta.containsProperty(StorageConstants.TIMEZONE);
- timezone = TimeZone.getTimeZone(meta.getProperty(StorageConstants.TIMEZONE, TajoConstants.DEFAULT_SYSTEM_TIMEZONE));
+ tableTimezone = TimeZone.getTimeZone(meta.getProperty(StorageConstants.TIMEZONE,
+ StorageUtil.TAJO_CONF.getSystemTimezone().getID()));
}
private static boolean isNull(ByteBuf val, ByteBuf nullBytes) {
@@ -62,7 +62,7 @@
}
private static boolean isNullText(ByteBuf val, ByteBuf nullBytes) {
- return val.readableBytes() > 0 && nullBytes.equals(val);
+ return val.readableBytes() > 0 && ByteBufUtil.equals(nullBytes, val);
}
@Override
@@ -123,20 +123,15 @@
out.write(bytes);
break;
case TIME:
- if (hasTimezone) {
- bytes = TimeDatum.asChars(tuple.getTimeDate(columnIndex), timezone, false).getBytes(Bytes.UTF8_CHARSET);
- } else {
- bytes = tuple.getTextBytes(columnIndex);
- }
+ bytes = tuple.getTextBytes(columnIndex);
length = bytes.length;
out.write(bytes);
break;
case TIMESTAMP:
- if (hasTimezone) {
- bytes = TimestampDatum.asChars(tuple.getTimeDate(columnIndex), timezone, false).getBytes(Bytes.UTF8_CHARSET);
- } else {
- bytes = tuple.getTextBytes(columnIndex);
- }
+ // UTC to table timezone
+ bytes = TimestampDatum.asChars(
+ tuple.getTimeDate(columnIndex), tableTimezone, false).getBytes(Bytes.UTF8_CHARSET);
+
length = bytes.length;
out.write(bytes);
break;
@@ -216,22 +211,13 @@
decoder.decode(buf.nioBuffer(buf.readerIndex(), buf.readableBytes())).toString());
break;
case TIME:
- if (hasTimezone) {
- datum = DatumFactory.createTime(
- decoder.decode(buf.nioBuffer(buf.readerIndex(), buf.readableBytes())).toString(), timezone);
- } else {
- datum = DatumFactory.createTime(
- decoder.decode(buf.nioBuffer(buf.readerIndex(), buf.readableBytes())).toString());
- }
+ datum = DatumFactory.createTime(
+ decoder.decode(buf.nioBuffer(buf.readerIndex(), buf.readableBytes())).toString());
break;
case TIMESTAMP:
- if (hasTimezone) {
- datum = DatumFactory.createTimestamp(
- decoder.decode(buf.nioBuffer(buf.readerIndex(), buf.readableBytes())).toString(), timezone);
- } else {
- datum = DatumFactory.createTimestamp(
- decoder.decode(buf.nioBuffer(buf.readerIndex(), buf.readableBytes())).toString());
- }
+ // Convert to UTC by table timezone
+ datum = DatumFactory.createTimestamp(
+ decoder.decode(buf.nioBuffer(buf.readerIndex(), buf.readableBytes())).toString(), tableTimezone);
break;
case INTERVAL:
datum = DatumFactory.createInterval(
diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/thirdparty/orc/TreeReaderFactory.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/thirdparty/orc/TreeReaderFactory.java
index 6ab630a..81c7f06 100644
--- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/thirdparty/orc/TreeReaderFactory.java
+++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/thirdparty/orc/TreeReaderFactory.java
@@ -37,7 +37,6 @@
import java.io.EOFException;
import java.io.IOException;
import java.io.InputStream;
-import java.sql.Timestamp;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.HashMap;
@@ -753,7 +752,6 @@
private final TimeZone readerTimeZone;
private TimeZone writerTimeZone;
private boolean hasSameTZRules;
- private final TimeZone timeZone;
TimestampTreeReader(TimeZone timeZone, int columnId, boolean skipCorrupt) throws IOException {
this(timeZone, columnId, null, null, null, null, skipCorrupt);
@@ -765,8 +763,8 @@
super(columnId, presentStream);
this.skipCorrupt = skipCorrupt;
this.baseTimestampMap = new HashMap<>();
- this.readerTimeZone = TimeZone.getDefault();
- this.writerTimeZone = readerTimeZone;
+ this.readerTimeZone = timeZone;
+ this.writerTimeZone = TimeZone.getDefault();
this.hasSameTZRules = writerTimeZone.hasSameRules(readerTimeZone);
this.base_timestamp = getBaseTimestamp(readerTimeZone.getID());
if (encoding != null) {
@@ -780,7 +778,6 @@
this.nanos = createIntegerReader(encoding.getKind(), nanosStream, false, skipCorrupt);
}
}
- this.timeZone = timeZone;
}
@Override
@@ -803,7 +800,7 @@
nanos = createIntegerReader(stripeFooter.getColumnsList().get(columnId).getKind(),
streams.get(new org.apache.orc.impl.StreamName(columnId,
OrcProto.Stream.Kind.SECONDARY)), false, skipCorrupt);
- getBaseTimestamp(stripeFooter.getWriterTimezone());
+ base_timestamp = getBaseTimestamp(stripeFooter.getWriterTimezone());
}
private long getBaseTimestamp(String timeZoneId) throws IOException {
@@ -849,8 +846,7 @@
if (valuePresent) {
long millis = decodeTimestamp(data.next(), nanos.next(), base_timestamp);
- long adjustedMillis = millis - writerTimeZone.getRawOffset();
- return DatumFactory.createTimestamp(DateTimeUtil.javaTimeToJulianTime(adjustedMillis));
+ return DatumFactory.createTimestamp(DateTimeUtil.javaTimeToJulianTime(millis));
} else {
return NullDatum.get();
}
diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/thirdparty/orc/WriterImpl.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/thirdparty/orc/WriterImpl.java
index e0ad3d7..12a155b 100644
--- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/thirdparty/orc/WriterImpl.java
+++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/thirdparty/orc/WriterImpl.java
@@ -50,6 +50,8 @@
import java.lang.management.ManagementFactory;
import java.nio.ByteBuffer;
import java.sql.Timestamp;
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
import java.util.*;
import static com.google.common.base.Preconditions.checkArgument;
@@ -1459,10 +1461,17 @@
this.nanos = createIntegerWriter(writer.createStream(id,
OrcProto.Stream.Kind.SECONDARY), false, isDirectV2, writer);
recordPosition(rowIndexPosition);
- // for unit tests to set different time zones
- this.base_timestamp = Timestamp.valueOf(BASE_TIMESTAMP_STRING).getTime() / DateTimeConstants.MSECS_PER_SEC;
writer.useWriterTimeZone(true);
timeZone = writer.getTimeZone();
+
+ SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
+ sdf.setTimeZone(timeZone);
+
+ try {
+ this.base_timestamp = sdf.parse(BASE_TIMESTAMP_STRING).getTime() / DateTimeConstants.MSECS_PER_SEC;
+ } catch (ParseException e) {
+ throw new IOException("Unable to create base timestamp", e);
+ }
}
@Override
@@ -1480,11 +1489,8 @@
super.write(datum);
if (datum != null && datum.isNotNull()) {
long javaTimestamp = DateTimeUtil.julianTimeToJavaTime(datum.asInt8());
-
- // revise timestamp value depends on timezone
- javaTimestamp += timeZone.getRawOffset();
-
Timestamp val = new Timestamp(javaTimestamp);
+
indexStatistics.updateTimestamp(val);
seconds.write((val.getTime() / DateTimeConstants.MSECS_PER_SEC) - base_timestamp);
nanos.write(formatNanos(val.getNanos()));
diff --git a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestCompressionStorages.java b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestCompressionStorages.java
index e9e6608..82bdf8c 100644
--- a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestCompressionStorages.java
+++ b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestCompressionStorages.java
@@ -120,7 +120,7 @@
.add("name", Type.TEXT)
.build();
- TableMeta meta = CatalogUtil.newTableMeta(dataFormat);
+ TableMeta meta = CatalogUtil.newTableMeta(dataFormat, conf);
meta.putProperty("compression.codec", codec.getCanonicalName());
meta.putProperty("compression.type", SequenceFile.CompressionType.BLOCK.name());
meta.putProperty("rcfile.serde", TextSerializerDeserializer.class.getName());
diff --git a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestDelimitedTextFile.java b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestDelimitedTextFile.java
index 29d8197..0c8695c 100644
--- a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestDelimitedTextFile.java
+++ b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestDelimitedTextFile.java
@@ -23,6 +23,7 @@
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.tajo.BuiltinStorages;
import org.apache.tajo.catalog.CatalogUtil;
import org.apache.tajo.catalog.Schema;
import org.apache.tajo.catalog.SchemaBuilder;
@@ -42,6 +43,7 @@
public class TestDelimitedTextFile {
private static final Log LOG = LogFactory.getLog(TestDelimitedTextFile.class);
+ private static TajoConf conf = new TajoConf();
private static Schema schema;
private static Tuple baseTuple;
@@ -89,7 +91,7 @@
@Test
public void testStripQuote() throws IOException, CloneNotSupportedException {
- TableMeta meta = CatalogUtil.newTableMeta("TEXT");
+ TableMeta meta = CatalogUtil.newTableMeta(BuiltinStorages.TEXT, conf);
meta.putProperty(StorageUtil.TEXT_DELIMITER, ",");
meta.putProperty(StorageUtil.QUOTE_CHAR, "\"");
FileFragment fragment = getFileFragment("testStripQuote.txt");
@@ -108,7 +110,7 @@
@Test
public void testIncompleteQuote() throws IOException, CloneNotSupportedException {
- TableMeta meta = CatalogUtil.newTableMeta("TEXT");
+ TableMeta meta = CatalogUtil.newTableMeta(BuiltinStorages.TEXT, conf);
meta.putProperty(StorageUtil.TEXT_DELIMITER, ",");
meta.putProperty(StorageUtil.QUOTE_CHAR, "\"");
FileFragment fragment = getFileFragment("testIncompleteQuote.txt");
@@ -127,7 +129,7 @@
@Test
public void testIgnoreAllErrors() throws IOException {
- TableMeta meta = CatalogUtil.newTableMeta("JSON");
+ TableMeta meta = CatalogUtil.newTableMeta(BuiltinStorages.JSON, conf);;
meta.putProperty(StorageUtil.TEXT_ERROR_TOLERANCE_MAXNUM, "-1");
FileFragment fragment = getFileFragment("testErrorTolerance1.json");
Scanner scanner = TablespaceManager.getLocalFs().getScanner(meta, schema, fragment, null);
@@ -145,7 +147,7 @@
@Test
public void testIgnoreOneErrorTolerance() throws IOException {
- TableMeta meta = CatalogUtil.newTableMeta("JSON");
+ TableMeta meta = CatalogUtil.newTableMeta(BuiltinStorages.JSON, conf);;
meta.putProperty(StorageUtil.TEXT_ERROR_TOLERANCE_MAXNUM, "1");
FileFragment fragment = getFileFragment("testErrorTolerance1.json");
Scanner scanner = TablespaceManager.getLocalFs().getScanner(meta, schema, fragment, null);
@@ -166,7 +168,7 @@
@Test
public void testNoErrorTolerance() throws IOException {
- TableMeta meta = CatalogUtil.newTableMeta("JSON");
+ TableMeta meta = CatalogUtil.newTableMeta(BuiltinStorages.JSON, conf);;
meta.putProperty(StorageUtil.TEXT_ERROR_TOLERANCE_MAXNUM, "0");
FileFragment fragment = getFileFragment("testErrorTolerance2.json");
Scanner scanner = TablespaceManager.getLocalFs().getScanner(meta, schema, fragment, null);
@@ -185,7 +187,7 @@
@Test
public void testIgnoreTruncatedValueErrorTolerance() throws IOException {
TajoConf conf = new TajoConf();
- TableMeta meta = CatalogUtil.newTableMeta("JSON");
+ TableMeta meta = CatalogUtil.newTableMeta(BuiltinStorages.JSON, conf);;
meta.putProperty(StorageUtil.TEXT_ERROR_TOLERANCE_MAXNUM, "1");
FileFragment fragment = getFileFragment("testErrorTolerance3.json");
Scanner scanner = TablespaceManager.getLocalFs().getScanner(meta, schema, fragment, null);
@@ -201,7 +203,7 @@
@Test
public void testSkippingHeaderWithJson() throws IOException {
- TableMeta meta = CatalogUtil.newTableMeta("JSON");
+ TableMeta meta = CatalogUtil.newTableMeta(BuiltinStorages.JSON, conf);;
meta.putProperty(StorageConstants.TEXT_SKIP_HEADER_LINE, "2");
FileFragment fragment = getFileFragment("testNormal.json");
Scanner scanner = TablespaceManager.getLocalFs().getScanner(meta, schema, fragment, null);
@@ -227,7 +229,7 @@
@Test
public void testSkippingHeaderWithText() throws IOException {
- TableMeta meta = CatalogUtil.newTableMeta("TEXT");
+ TableMeta meta = CatalogUtil.newTableMeta(BuiltinStorages.TEXT, conf);
meta.putProperty(StorageConstants.TEXT_SKIP_HEADER_LINE, "1");
meta.putProperty(StorageConstants.TEXT_DELIMITER, ",");
FileFragment fragment = getFileFragment("testSkip.txt");
diff --git a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestFileSystems.java b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestFileSystems.java
index 48740fb..c62a01c 100644
--- a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestFileSystems.java
+++ b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestFileSystems.java
@@ -22,6 +22,7 @@
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.LocalFileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.tajo.BuiltinStorages;
import org.apache.tajo.catalog.CatalogUtil;
import org.apache.tajo.catalog.Schema;
import org.apache.tajo.catalog.SchemaBuilder;
@@ -104,7 +105,7 @@
.add("name", Type.TEXT)
.build();
- TableMeta meta = CatalogUtil.newTableMeta("TEXT");
+ TableMeta meta = CatalogUtil.newTableMeta(BuiltinStorages.TEXT, conf);
Tuple[] tuples = new Tuple[4];
for (int i = 0; i < tuples.length; i++) {
diff --git a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestFileTablespace.java b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestFileTablespace.java
index 65453dd..dc8781e 100644
--- a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestFileTablespace.java
+++ b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestFileTablespace.java
@@ -88,7 +88,7 @@
.add("name", Type.TEXT)
.build();
- TableMeta meta = CatalogUtil.newTableMeta(BuiltinStorages.TEXT);
+ TableMeta meta = CatalogUtil.newTableMeta(BuiltinStorages.TEXT, conf);
VTuple[] tuples = new VTuple[4];
for(int i=0; i < tuples.length; i++) {
@@ -151,7 +151,8 @@
.add("age", Type.INT4)
.add("name", Type.TEXT)
.build();
- TableMeta meta = CatalogUtil.newTableMeta(BuiltinStorages.TEXT);
+
+ TableMeta meta = CatalogUtil.newTableMeta(BuiltinStorages.TEXT, conf);
List<Fragment> splits = Lists.newArrayList();
// Get FileFragments in partition batch
@@ -206,7 +207,8 @@
.add("age", Type.INT4)
.add("name", Type.TEXT)
.build();
- TableMeta meta = CatalogUtil.newTableMeta(BuiltinStorages.TEXT);
+
+ TableMeta meta = CatalogUtil.newTableMeta(BuiltinStorages.TEXT, conf);
List<Fragment> splits = Lists.newArrayList();
// Get FileFragments in partition batch
@@ -250,7 +252,8 @@
.add("age", Type.INT4)
.add("name", Type.TEXT)
.build();
- TableMeta meta = CatalogUtil.newTableMeta(BuiltinStorages.TEXT);
+
+ TableMeta meta = CatalogUtil.newTableMeta(BuiltinStorages.TEXT, conf);
List<Fragment> splits = Lists.newArrayList();
splits.addAll(sm.getSplits("data", meta, schema, tablePath));
diff --git a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestLineReader.java b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestLineReader.java
index b72c98a..541e12d 100644
--- a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestLineReader.java
+++ b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestLineReader.java
@@ -25,6 +25,7 @@
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.io.compress.DeflateCodec;
+import org.apache.tajo.BuiltinStorages;
import org.apache.tajo.catalog.CatalogUtil;
import org.apache.tajo.catalog.Schema;
import org.apache.tajo.catalog.SchemaBuilder;
@@ -66,7 +67,7 @@
.add("comment2", Type.TEXT)
.build();
- TableMeta meta = CatalogUtil.newTableMeta("TEXT");
+ TableMeta meta = CatalogUtil.newTableMeta(BuiltinStorages.TEXT, conf);
Path tablePath = new Path(testDir, "line.data");
FileAppender appender = (FileAppender) TablespaceManager.getLocalFs().getAppender(
null, null, meta, schema, tablePath);
@@ -119,7 +120,7 @@
.add("comment2", Type.TEXT)
.build();
- TableMeta meta = CatalogUtil.newTableMeta("TEXT");
+ TableMeta meta = CatalogUtil.newTableMeta(BuiltinStorages.TEXT, conf);
meta.putProperty("compression.codec", DeflateCodec.class.getCanonicalName());
Path tablePath = new Path(testDir, "testLineDelimitedReaderWithCompression." + DeflateCodec.class.getSimpleName());
@@ -179,7 +180,7 @@
.add("comment2", Type.TEXT)
.build();
- TableMeta meta = CatalogUtil.newTableMeta("TEXT");
+ TableMeta meta = CatalogUtil.newTableMeta(BuiltinStorages.TEXT, conf);
Path tablePath = new Path(testDir, "testLineDelimitedReader");
FileAppender appender = (FileAppender) TablespaceManager.getLocalFs().getAppender(
@@ -286,7 +287,7 @@
.add("comment2", Type.TEXT)
.build();
- TableMeta meta = CatalogUtil.newTableMeta("TEXT");
+ TableMeta meta = CatalogUtil.newTableMeta(BuiltinStorages.TEXT, conf);
Path tablePath = new Path(testDir, "testSeekableByteBufLineReader.data");
FileAppender appender = (FileAppender) TablespaceManager.getLocalFs().getAppender(
null, null, meta, schema, tablePath);
diff --git a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestMergeScanner.java b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestMergeScanner.java
index eae8e15..a99dd3c 100644
--- a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestMergeScanner.java
+++ b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestMergeScanner.java
@@ -33,7 +33,6 @@
import org.apache.tajo.storage.fragment.FileFragment;
import org.apache.tajo.storage.fragment.Fragment;
import org.apache.tajo.util.CommonTestingUtil;
-import org.apache.tajo.util.KeyValueSet;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
@@ -106,9 +105,8 @@
.add("age", Type.INT8)
.build();
- KeyValueSet options = new KeyValueSet();
- TableMeta meta = CatalogUtil.newTableMeta(dataFormat, options);
- meta.setPropertySet(CatalogUtil.newDefaultProperty(dataFormat));
+ TableMeta meta = CatalogUtil.newTableMeta(dataFormat, conf);
+
if (dataFormat.equalsIgnoreCase("AVRO")) {
meta.putProperty(StorageConstants.AVRO_SCHEMA_LITERAL, TEST_MULTIPLE_FILES_AVRO_SCHEMA);
}
diff --git a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestStorages.java b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestStorages.java
index bcbb6c3..bee692f 100644
--- a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestStorages.java
+++ b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestStorages.java
@@ -48,7 +48,6 @@
import org.apache.tajo.storage.sequencefile.SequenceFileScanner;
import org.apache.tajo.util.CommonTestingUtil;
import org.apache.tajo.util.JavaResourceUtil;
-import org.apache.tajo.util.KeyValueSet;
import org.junit.After;
import org.junit.Test;
import org.junit.runner.RunWith;
@@ -192,7 +191,7 @@
.add("age", Type.INT8)
.build();
- TableMeta meta = CatalogUtil.newTableMeta(dataFormat);
+ TableMeta meta = CatalogUtil.newTableMeta(dataFormat, conf);
Path tablePath = new Path(testDir, "Splitable.data");
FileTablespace sm = TablespaceManager.getLocalFs();
Appender appender = sm.getAppender(meta, schema, tablePath);
@@ -248,8 +247,7 @@
.add("score", Type.FLOAT4)
.build();
- TableMeta meta = CatalogUtil.newTableMeta(dataFormat);
- meta.setPropertySet(CatalogUtil.newDefaultProperty(dataFormat));
+ TableMeta meta = CatalogUtil.newTableMeta(dataFormat, conf);
if (dataFormat.equalsIgnoreCase(BuiltinStorages.AVRO)) {
meta.putProperty(StorageConstants.AVRO_SCHEMA_LITERAL,
TEST_PROJECTION_AVRO_SCHEMA);
@@ -292,7 +290,7 @@
.add("age", Type.INT8)
.build();
- TableMeta meta = CatalogUtil.newTableMeta(dataFormat);
+ TableMeta meta = CatalogUtil.newTableMeta(dataFormat, conf);
Path tablePath = new Path(testDir, "Splitable.data");
FileTablespace sm = TablespaceManager.getLocalFs();
Appender appender = sm.getAppender(meta, schema, tablePath);
@@ -348,8 +346,7 @@
.add("score", Type.FLOAT4)
.build();
- TableMeta meta = CatalogUtil.newTableMeta(dataFormat);
- meta.setPropertySet(CatalogUtil.newDefaultProperty(dataFormat));
+ TableMeta meta = CatalogUtil.newTableMeta(dataFormat, conf);
if (dataFormat.equalsIgnoreCase(BuiltinStorages.AVRO)) {
meta.putProperty(StorageConstants.AVRO_SCHEMA_LITERAL,
TEST_PROJECTION_AVRO_SCHEMA);
@@ -424,9 +421,7 @@
Schema schema = schemaBld.build();
- KeyValueSet options = new KeyValueSet();
- TableMeta meta = CatalogUtil.newTableMeta(dataFormat, options);
- meta.setPropertySet(CatalogUtil.newDefaultProperty(dataFormat));
+ TableMeta meta = CatalogUtil.newTableMeta(dataFormat, conf);
if (dataFormat.equalsIgnoreCase(BuiltinStorages.AVRO)) {
String path = JavaResourceUtil.getResourceURL("dataset/testVariousTypes.avsc").toString();
meta.putProperty(StorageConstants.AVRO_SCHEMA_URL, path);
@@ -496,9 +491,7 @@
Schema schema = schemaBld.build();
- KeyValueSet options = new KeyValueSet();
- TableMeta meta = CatalogUtil.newTableMeta(dataFormat, options);
- meta.setPropertySet(CatalogUtil.newDefaultProperty(dataFormat));
+ TableMeta meta = CatalogUtil.newTableMeta(dataFormat, conf);
meta.putProperty(StorageConstants.TEXT_NULL, "\\\\N");
meta.putProperty(StorageConstants.RCFILE_NULL, "\\\\N");
meta.putProperty(StorageConstants.RCFILE_SERDE, TextSerializerDeserializer.class.getName());
@@ -593,9 +586,7 @@
Schema schema = schemaBld.build();
- KeyValueSet options = new KeyValueSet();
- TableMeta meta = CatalogUtil.newTableMeta(dataFormat, options);
- meta.setPropertySet(CatalogUtil.newDefaultProperty(dataFormat));
+ TableMeta meta = CatalogUtil.newTableMeta(dataFormat, conf);
meta.putProperty(StorageConstants.TEXT_NULL, "\\\\N");
meta.putProperty(StorageConstants.RCFILE_NULL, "\\\\N");
meta.putProperty(StorageConstants.RCFILE_SERDE, TextSerializerDeserializer.class.getName());
@@ -698,8 +689,7 @@
.add("col11", CatalogUtil.newDataType(Type.PROTOBUF, TajoIdProtos.QueryIdProto.class.getName()))
.build();
- KeyValueSet options = new KeyValueSet();
- TableMeta meta = CatalogUtil.newTableMeta(dataFormat, options);
+ TableMeta meta = CatalogUtil.newTableMeta(dataFormat, conf);
meta.putProperty(StorageConstants.CSVFILE_SERDE, TextSerializerDeserializer.class.getName());
Path tablePath = new Path(testDir, "testRCFileTextSerializeDeserialize.data");
@@ -764,8 +754,7 @@
.add("col11", CatalogUtil.newDataType(Type.PROTOBUF, TajoIdProtos.QueryIdProto.class.getName()))
.build();
- KeyValueSet options = new KeyValueSet();
- TableMeta meta = CatalogUtil.newTableMeta(dataFormat, options);
+ TableMeta meta = CatalogUtil.newTableMeta(dataFormat, conf);
meta.putProperty(StorageConstants.RCFILE_SERDE, BinarySerializerDeserializer.class.getName());
Path tablePath = new Path(testDir, "testRCFileBinarySerializeDeserialize.data");
@@ -829,8 +818,7 @@
.add("col10", Type.INET4)
.add("col11", CatalogUtil.newDataType(Type.PROTOBUF, TajoIdProtos.QueryIdProto.class.getName())).build();
- KeyValueSet options = new KeyValueSet();
- TableMeta meta = CatalogUtil.newTableMeta(dataFormat, options);
+ TableMeta meta = CatalogUtil.newTableMeta(dataFormat, conf);
meta.putProperty(StorageConstants.SEQUENCEFILE_SERDE, TextSerializerDeserializer.class.getName());
Path tablePath = new Path(testDir, "testSequenceFileTextSerializeDeserialize.data");
@@ -898,8 +886,7 @@
.add("col11", CatalogUtil.newDataType(Type.PROTOBUF, TajoIdProtos.QueryIdProto.class.getName()))
.build();
- KeyValueSet options = new KeyValueSet();
- TableMeta meta = CatalogUtil.newTableMeta(dataFormat, options);
+ TableMeta meta = CatalogUtil.newTableMeta(dataFormat, conf);
meta.putProperty(StorageConstants.SEQUENCEFILE_SERDE, BinarySerializerDeserializer.class.getName());
Path tablePath = new Path(testDir, "testVariousTypes.data");
@@ -963,8 +950,8 @@
schema.add("col" + index++, Type.TIME);
}
- KeyValueSet options = new KeyValueSet();
- TableMeta meta = CatalogUtil.newTableMeta(dataFormat, options);
+ TableMeta meta = CatalogUtil.newTableMeta(dataFormat, conf);
+ assertTrue(meta.containsProperty(StorageConstants.TIMEZONE));
Path tablePath = new Path(testDir, "testTime.data");
FileTablespace sm = TablespaceManager.getLocalFs();
@@ -1011,7 +998,7 @@
.add("comment", Type.TEXT)
.build();
- TableMeta meta = CatalogUtil.newTableMeta(dataFormat);
+ TableMeta meta = CatalogUtil.newTableMeta(dataFormat, conf);
Path tablePath = new Path(testDir, "Seekable.data");
FileTablespace sm = TablespaceManager.getLocalFs();
FileAppender appender = (FileAppender) sm.getAppender(meta, schema, tablePath);
@@ -1089,8 +1076,7 @@
.add("col5", Type.INT8)
.build();
- KeyValueSet options = new KeyValueSet();
- TableMeta meta = CatalogUtil.newTableMeta(dataFormat, options);
+ TableMeta meta = CatalogUtil.newTableMeta(dataFormat, conf);
if (dataFormat.equalsIgnoreCase(BuiltinStorages.AVRO)) {
meta.putProperty(StorageConstants.AVRO_SCHEMA_LITERAL, TEST_MAX_VALUE_AVRO_SCHEMA);
}
@@ -1155,9 +1141,7 @@
.add("col3", Type.INT2)
.build();
- KeyValueSet options = new KeyValueSet();
- TableMeta meta = CatalogUtil.newTableMeta(dataFormat, options);
- meta.setPropertySet(CatalogUtil.newDefaultProperty(dataFormat));
+ TableMeta meta = CatalogUtil.newTableMeta(dataFormat, conf);
Path tablePath = new Path(testDir, "testLessThanSchemaSize.data");
FileTablespace sm = TablespaceManager.getLocalFs();
@@ -1221,9 +1205,7 @@
.add("col1", Type.CHAR)
.build();
- KeyValueSet options = new KeyValueSet();
- TableMeta meta = CatalogUtil.newTableMeta(dataFormat, options);
- meta.setPropertySet(CatalogUtil.newDefaultProperty(dataFormat));
+ TableMeta meta = CatalogUtil.newTableMeta(dataFormat, conf);
Path tablePath = new Path(testDir, "test_dataformat_oversize.data");
FileTablespace sm = TablespaceManager.getLocalFs();
@@ -1263,8 +1245,7 @@
Schema schema = SchemaBuilder.builder().add("col1", Type.TEXT).build();
- KeyValueSet options = new KeyValueSet();
- TableMeta meta = CatalogUtil.newTableMeta(dataFormat, options);
+ TableMeta meta = CatalogUtil.newTableMeta(dataFormat, conf);
FileTablespace sm = TablespaceManager.getLocalFs();
Path tablePath = new Path(testDir, "testTextHandling.data");
@@ -1307,8 +1288,7 @@
.add("score", Type.FLOAT4)
.build();
- TableMeta meta = CatalogUtil.newTableMeta(dataFormat);
- meta.setPropertySet(CatalogUtil.newDefaultProperty(dataFormat));
+ TableMeta meta = CatalogUtil.newTableMeta(dataFormat, conf);
if (dataFormat.equalsIgnoreCase(BuiltinStorages.AVRO)) {
meta.putProperty(StorageConstants.AVRO_SCHEMA_LITERAL,
TEST_PROJECTION_AVRO_SCHEMA);
@@ -1345,8 +1325,7 @@
.add("col5", Type.INT8)
.build();
- KeyValueSet options = new KeyValueSet();
- TableMeta meta = CatalogUtil.newTableMeta(dataFormat, options);
+ TableMeta meta = CatalogUtil.newTableMeta(dataFormat, conf);
if (dataFormat.equalsIgnoreCase(BuiltinStorages.AVRO)) {
meta.putProperty(StorageConstants.AVRO_SCHEMA_LITERAL, TEST_MAX_VALUE_AVRO_SCHEMA);
}
@@ -1392,8 +1371,7 @@
.add("score", Type.FLOAT4)
.build();
- TableMeta meta = CatalogUtil.newTableMeta(dataFormat);
- meta.setPropertySet(CatalogUtil.newDefaultProperty(dataFormat));
+ TableMeta meta = CatalogUtil.newTableMeta(dataFormat, conf);
if (dataFormat.equalsIgnoreCase(BuiltinStorages.AVRO)) {
meta.putProperty(StorageConstants.AVRO_SCHEMA_LITERAL,
TEST_PROJECTION_AVRO_SCHEMA);
diff --git a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/avro/TestAvroUtil.java b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/avro/TestAvroUtil.java
index ddeb340..705255d 100644
--- a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/avro/TestAvroUtil.java
+++ b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/avro/TestAvroUtil.java
@@ -19,6 +19,7 @@
package org.apache.tajo.storage.avro;
import org.apache.avro.Schema;
+import org.apache.tajo.BuiltinStorages;
import org.apache.tajo.HttpFileServer;
import org.apache.tajo.catalog.CatalogUtil;
import org.apache.tajo.catalog.TableMeta;
@@ -58,12 +59,13 @@
@Test
public void testGetSchema() throws IOException, URISyntaxException {
- TableMeta meta = CatalogUtil.newTableMeta("AVRO");
+ TajoConf conf = new TajoConf();
+ TableMeta meta = CatalogUtil.newTableMeta(BuiltinStorages.AVRO, conf);
meta.putProperty(StorageConstants.AVRO_SCHEMA_LITERAL, FileUtil.readTextFile(new File(schemaUrl.getPath())));
- Schema schema = AvroUtil.getAvroSchema(meta, new TajoConf());
+ Schema schema = AvroUtil.getAvroSchema(meta, conf);
assertEquals(expected, schema);
- meta = CatalogUtil.newTableMeta("AVRO");
+ meta = CatalogUtil.newTableMeta(BuiltinStorages.AVRO, conf);
meta.putProperty(StorageConstants.AVRO_SCHEMA_URL, schemaUrl.getPath());
schema = AvroUtil.getAvroSchema(meta, new TajoConf());
assertEquals(expected, schema);
@@ -74,7 +76,7 @@
InetSocketAddress addr = server.getBindAddress();
String url = "http://127.0.0.1:" + addr.getPort() + schemaUrl.getPath();
- meta = CatalogUtil.newTableMeta("AVRO");
+ meta = CatalogUtil.newTableMeta(BuiltinStorages.AVRO, conf);
meta.putProperty(StorageConstants.AVRO_SCHEMA_URL, url);
schema = AvroUtil.getAvroSchema(meta, new TajoConf());
} finally {
diff --git a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/index/TestBSTIndex.java b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/index/TestBSTIndex.java
index 93052f2..0e6fde5 100644
--- a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/index/TestBSTIndex.java
+++ b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/index/TestBSTIndex.java
@@ -87,7 +87,7 @@
@Test
public void testFindValue() throws IOException {
- meta = CatalogUtil.newTableMeta(dataFormat);
+ meta = CatalogUtil.newTableMeta(dataFormat, conf);
Path tablePath = new Path(testDir, "testFindValue_" + dataFormat);
Appender appender = ((FileTablespace) TablespaceManager.getLocalFs())
@@ -179,7 +179,7 @@
@Test
public void testBuildIndexWithAppender() throws IOException {
- meta = CatalogUtil.newTableMeta(dataFormat);
+ meta = CatalogUtil.newTableMeta(dataFormat, conf);
Path tablePath = new Path(testDir, "testBuildIndexWithAppender_" + dataFormat);
FileAppender appender = (FileAppender) ((FileTablespace) TablespaceManager.getLocalFs())
@@ -260,7 +260,7 @@
@Test
public void testFindOmittedValue() throws IOException {
- meta = CatalogUtil.newTableMeta(dataFormat);
+ meta = CatalogUtil.newTableMeta(dataFormat, conf);
Path tablePath = StorageUtil.concatPath(testDir, "testFindOmittedValue_" + dataFormat);
Appender appender = ((FileTablespace) TablespaceManager.getLocalFs()).getAppender(meta, schema, tablePath);
@@ -332,7 +332,7 @@
@Test
public void testFindNextKeyValue() throws IOException {
- meta = CatalogUtil.newTableMeta(dataFormat);
+ meta = CatalogUtil.newTableMeta(dataFormat, conf);
Path tablePath = new Path(testDir, "testFindNextKeyValue_" + dataFormat);
Appender appender = ((FileTablespace) TablespaceManager.getLocalFs())
@@ -426,7 +426,7 @@
@Test
public void testFindNextKeyOmittedValue() throws IOException {
- meta = CatalogUtil.newTableMeta(dataFormat);
+ meta = CatalogUtil.newTableMeta(dataFormat, conf);
Path tablePath = new Path(testDir, "testFindNextKeyOmittedValue_" + dataFormat);
Appender appender = (((FileTablespace) TablespaceManager.getLocalFs()))
@@ -509,7 +509,7 @@
@Test
public void testFindMinValue() throws IOException {
- meta = CatalogUtil.newTableMeta(dataFormat);
+ meta = CatalogUtil.newTableMeta(dataFormat, conf);
Path tablePath = new Path(testDir, "testFindMinValue" + dataFormat);
Appender appender = ((FileTablespace) TablespaceManager.getLocalFs())
@@ -596,7 +596,7 @@
@Test
public void testMinMax() throws IOException {
- meta = CatalogUtil.newTableMeta(dataFormat);
+ meta = CatalogUtil.newTableMeta(dataFormat, conf);
Path tablePath = new Path(testDir, "testMinMax_" + dataFormat);
Appender appender = ((FileTablespace) TablespaceManager.getLocalFs())
@@ -703,7 +703,7 @@
@Test
public void testConcurrentAccess() throws IOException, InterruptedException {
- meta = CatalogUtil.newTableMeta(dataFormat);
+ meta = CatalogUtil.newTableMeta(dataFormat, conf);
Path tablePath = new Path(testDir, "testConcurrentAccess_" + dataFormat);
Appender appender = ((FileTablespace) TablespaceManager.getLocalFs())
@@ -786,7 +786,7 @@
@Test
public void testFindValueDescOrder() throws IOException {
- meta = CatalogUtil.newTableMeta(dataFormat);
+ meta = CatalogUtil.newTableMeta(dataFormat, conf);
Path tablePath = new Path(testDir, "testFindValueDescOrder_" + dataFormat);
Appender appender = ((FileTablespace) TablespaceManager.getLocalFs())
@@ -881,7 +881,7 @@
@Test
public void testFindNextKeyValueDescOrder() throws IOException {
- meta = CatalogUtil.newTableMeta(dataFormat);
+ meta = CatalogUtil.newTableMeta(dataFormat, conf);
Path tablePath = new Path(testDir, "testFindNextKeyValueDescOrder_" + dataFormat);
Appender appender = ((FileTablespace) TablespaceManager.getLocalFs()).getAppender(meta, schema, tablePath);
@@ -980,7 +980,7 @@
@Test
public void testFindValueASCOrder() throws IOException {
- meta = CatalogUtil.newTableMeta(dataFormat);
+ meta = CatalogUtil.newTableMeta(dataFormat, conf);
Path tablePath = new Path(testDir, "testFindValue_" + dataFormat);
Appender appender = ((FileTablespace) TablespaceManager.getLocalFs())
diff --git a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/index/TestSingleCSVFileBSTIndex.java b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/index/TestSingleCSVFileBSTIndex.java
index 358be38..4f6b566 100644
--- a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/index/TestSingleCSVFileBSTIndex.java
+++ b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/index/TestSingleCSVFileBSTIndex.java
@@ -73,7 +73,7 @@
@Test
public void testFindValueInSingleCSV() throws IOException {
- meta = CatalogUtil.newTableMeta(BuiltinStorages.TEXT);
+ meta = CatalogUtil.newTableMeta(BuiltinStorages.TEXT, conf);
Path tablePath = StorageUtil.concatPath(testDir, "testFindValueInSingleCSV", "table.csv");
fs.mkdirs(tablePath.getParent());
@@ -165,7 +165,7 @@
@Test
public void testFindNextKeyValueInSingleCSV() throws IOException {
- meta = CatalogUtil.newTableMeta(BuiltinStorages.TEXT);
+ meta = CatalogUtil.newTableMeta(BuiltinStorages.TEXT, conf);
Path tablePath = StorageUtil.concatPath(testDir, "testFindNextKeyValueInSingleCSV",
"table1.csv");
diff --git a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/json/TestJsonSerDe.java b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/json/TestJsonSerDe.java
index 30367c2..8cf4586 100644
--- a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/json/TestJsonSerDe.java
+++ b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/json/TestJsonSerDe.java
@@ -21,6 +21,7 @@
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.tajo.BuiltinStorages;
import org.apache.tajo.catalog.CatalogUtil;
import org.apache.tajo.catalog.Schema;
import org.apache.tajo.catalog.SchemaBuilder;
@@ -70,7 +71,7 @@
public void testVarioutType() throws IOException {
TajoConf conf = new TajoConf();
- TableMeta meta = CatalogUtil.newTableMeta("JSON");
+ TableMeta meta = CatalogUtil.newTableMeta(BuiltinStorages.JSON, conf);
Path tablePath = new Path(getResourcePath("dataset", "TestJsonSerDe"), "testVariousType.json");
FileSystem fs = FileSystem.getLocal(conf);
FileStatus status = fs.getFileStatus(tablePath);
@@ -104,7 +105,7 @@
public void testUnicodeWithControlChar() throws IOException {
TajoConf conf = new TajoConf();
- TableMeta meta = CatalogUtil.newTableMeta("JSON");
+ TableMeta meta = CatalogUtil.newTableMeta(BuiltinStorages.JSON, conf);
Path tablePath = new Path(getResourcePath("dataset", "TestJsonSerDe"), "testUnicodeWithControlChar.json");
FileSystem fs = FileSystem.getLocal(conf);
FileStatus status = fs.getFileStatus(tablePath);
diff --git a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/raw/TestDirectRawFile.java b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/raw/TestDirectRawFile.java
index 19e3e95..569438e 100644
--- a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/raw/TestDirectRawFile.java
+++ b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/raw/TestDirectRawFile.java
@@ -172,7 +172,7 @@
MemoryRowBlock rowBlock = createRowBlock(rowNum);
- TableMeta meta = CatalogUtil.newTableMeta(BuiltinStorages.DRAW);
+ TableMeta meta = CatalogUtil.newTableMeta(BuiltinStorages.DRAW, tajoConf);
FileStatus outputFile = writeRowBlock(tajoConf, meta, rowBlock);
rowBlock.release();
@@ -201,7 +201,7 @@
int rowNum = 2;
MemoryRowBlock rowBlock = createRowBlock(rowNum);
- TableMeta meta = CatalogUtil.newTableMeta(BuiltinStorages.DRAW);
+ TableMeta meta = CatalogUtil.newTableMeta(BuiltinStorages.DRAW, tajoConf);
FileStatus outputFile = writeRowBlock(tajoConf, meta, rowBlock);
rowBlock.release();
@@ -230,7 +230,7 @@
MemoryRowBlock rowBlock = createRowBlock(rowNum);
- TableMeta meta = CatalogUtil.newTableMeta(BuiltinStorages.DRAW);
+ TableMeta meta = CatalogUtil.newTableMeta(BuiltinStorages.DRAW, tajoConf);
FileStatus outputFile = writeRowBlock(tajoConf, meta, rowBlock);
rowBlock.release();
