PIG-5431: Date datatype is different between Hive 1.x and Hive 3.x (rohini)
git-svn-id: https://svn.apache.org/repos/asf/pig/trunk@1906696 13f79535-47bb-0310-9956-ffa450edef68
diff --git a/CHANGES.txt b/CHANGES.txt
index a85b7fd..483bc95 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -116,6 +116,8 @@
BUG FIXES
+PIG-5431: Date datatype is different between Hive 1.x and Hive 3.x (rohini)
+
PIG-5433: Fix test failures with TestHBaseStorage and htrace dependency (rohini)
PIG-5430: TestTezGraceParallelism failing due to tez log change (knoguchi)
diff --git a/shims/src/hive1/org/apache/pig/hive/HiveShims.java b/shims/src/hive1/org/apache/pig/hive/HiveShims.java
index b606821..34bdb7e 100644
--- a/shims/src/hive1/org/apache/pig/hive/HiveShims.java
+++ b/shims/src/hive1/org/apache/pig/hive/HiveShims.java
@@ -19,6 +19,7 @@
import java.math.BigDecimal;
import java.math.BigInteger;
+import java.sql.Date;
import java.sql.Timestamp;
import org.apache.hadoop.hive.conf.HiveConf;
@@ -143,6 +144,17 @@
return new SimpleGenericUDAFParameterInfo(arguments, distinct, allColumns);
}
+ public static class DateShim {
+
+ public static Date cast(Object ts) {
+ return (Date) ts;
+ }
+
+ public static long millisFromDate(Object ts) {
+ return cast(ts).getTime();
+ }
+ }
+
public static class TimestampShim {
public static Timestamp cast(Object ts) {
diff --git a/shims/src/hive3/org/apache/pig/hive/HiveShims.java b/shims/src/hive3/org/apache/pig/hive/HiveShims.java
index 32456c5..063fbcd 100644
--- a/shims/src/hive3/org/apache/pig/hive/HiveShims.java
+++ b/shims/src/hive3/org/apache/pig/hive/HiveShims.java
@@ -20,6 +20,7 @@
import java.math.BigDecimal;
import java.math.BigInteger;
+import org.apache.hadoop.hive.common.type.Date;
import org.apache.hadoop.hive.common.type.HiveDecimal;
import org.apache.hadoop.hive.common.type.Timestamp;
import org.apache.hadoop.hive.conf.HiveConf;
@@ -165,6 +166,17 @@
return new SimpleGenericUDAFParameterInfo(arguments, false, distinct, allColumns);
}
+ public static class DateShim {
+
+ public static Date cast(Object ts) {
+ return (Date) ts;
+ }
+
+ public static long millisFromDate(Object ts) {
+ return cast(ts).toEpochMilli();
+ }
+ }
+
public static class TimestampShim {
public static Timestamp cast(Object ts) {
@@ -191,4 +203,4 @@
}
}
-}
\ No newline at end of file
+}
diff --git a/src/org/apache/pig/impl/util/hive/HiveUtils.java b/src/org/apache/pig/impl/util/hive/HiveUtils.java
index 5fc0f40..a725a03 100644
--- a/src/org/apache/pig/impl/util/hive/HiveUtils.java
+++ b/src/org/apache/pig/impl/util/hive/HiveUtils.java
@@ -184,8 +184,7 @@
result = new DateTime(HiveShims.TimestampShim.millisFromTimestamp(poi.getPrimitiveJavaObject(obj)));
break;
case DATE:
- java.sql.Date origDate = (java.sql.Date)poi.getPrimitiveJavaObject(obj);
- result = new DateTime(origDate.getTime());
+ result = new DateTime(HiveShims.DateShim.millisFromDate(poi.getPrimitiveJavaObject(obj)));
break;
case DECIMAL:
org.apache.hadoop.hive.common.type.HiveDecimal origDecimal =
diff --git a/test/org/apache/pig/builtin/TestOrcStorage.java b/test/org/apache/pig/builtin/TestOrcStorage.java
index b8f6bd5..5472e41 100644
--- a/test/org/apache/pig/builtin/TestOrcStorage.java
+++ b/test/org/apache/pig/builtin/TestOrcStorage.java
@@ -183,6 +183,20 @@
}
assertEquals(count, 10000);
}
+
+ @Test
+ public void testDate() throws Exception {
+ pigServer.registerQuery("A = load '" + basedir + "datetype.orc'" + " using OrcStorage();" );
+ Schema schema = pigServer.dumpSchema("A");
+ assertEquals(schema.size(), 1);
+ assertEquals(schema.getField(0).type, DataType.DATETIME);
+ Iterator<Tuple> iter = pigServer.openIterator("A");
+ Tuple t = iter.next();
+ assertTrue(t.get(0) instanceof DateTime);
+ assertEquals(t.toString(), "(2022-07-07T00:00:00.000Z)");
+ assertEquals(iter.next().toString(), "(2022-09-09T00:00:00.000Z)");
+ assertFalse(iter.hasNext());
+ }
@Test
// See PIG-4218
diff --git a/test/org/apache/pig/builtin/orc/datetype.orc b/test/org/apache/pig/builtin/orc/datetype.orc
new file mode 100644
index 0000000..391b27c
--- /dev/null
+++ b/test/org/apache/pig/builtin/orc/datetype.orc
Binary files differ
