ORC-526: Make orc-tools convert respect second fractions.
Fixes #425
Signed-off-by: Owen O'Malley <omalley@apache.org>
diff --git a/java/tools/src/java/org/apache/orc/tools/convert/CsvReader.java b/java/tools/src/java/org/apache/orc/tools/convert/CsvReader.java
index 62c458d..3967a49 100644
--- a/java/tools/src/java/org/apache/orc/tools/convert/CsvReader.java
+++ b/java/tools/src/java/org/apache/orc/tools/convert/CsvReader.java
@@ -254,11 +254,15 @@
dateTimeFormatter.parseBest(values[offset],
ZonedDateTime.FROM, LocalDateTime.FROM);
if (temporalAccessor instanceof ZonedDateTime) {
- vector.set(row, new Timestamp(
- ((ZonedDateTime) temporalAccessor).toEpochSecond() * 1000L));
+ ZonedDateTime zonedDateTime = ((ZonedDateTime) temporalAccessor);
+ Timestamp timestamp = new Timestamp(zonedDateTime.toEpochSecond() * 1000L);
+ timestamp.setNanos(zonedDateTime.getNano());
+ vector.set(row, timestamp);
} else if (temporalAccessor instanceof LocalDateTime) {
- vector.set(row, new Timestamp(((LocalDateTime) temporalAccessor)
- .atZone(ZoneId.systemDefault()).toEpochSecond() * 1000L));
+ ZonedDateTime tz = ((LocalDateTime) temporalAccessor).atZone(ZoneId.systemDefault());
+ Timestamp timestamp = new Timestamp(tz.toEpochSecond() * 1000L);
+ timestamp.setNanos(tz.getNano());
+ vector.set(row, timestamp);
} else {
column.noNulls = false;
column.isNull[row] = true;
diff --git a/java/tools/src/java/org/apache/orc/tools/convert/JsonReader.java b/java/tools/src/java/org/apache/orc/tools/convert/JsonReader.java
index 7db3c67..c3720af 100644
--- a/java/tools/src/java/org/apache/orc/tools/convert/JsonReader.java
+++ b/java/tools/src/java/org/apache/orc/tools/convert/JsonReader.java
@@ -147,9 +147,15 @@
TemporalAccessor temporalAccessor = dateTimeFormatter.parseBest(value.getAsString(),
ZonedDateTime.FROM, LocalDateTime.FROM);
if (temporalAccessor instanceof ZonedDateTime) {
- vector.set(row, new Timestamp(((ZonedDateTime) temporalAccessor).toEpochSecond() * 1000L));
+ ZonedDateTime zonedDateTime = ((ZonedDateTime) temporalAccessor);
+ Timestamp timestamp = new Timestamp(zonedDateTime.toEpochSecond() * 1000L);
+ timestamp.setNanos(zonedDateTime.getNano());
+ vector.set(row, timestamp);
} else if (temporalAccessor instanceof LocalDateTime) {
- vector.set(row, new Timestamp(((LocalDateTime) temporalAccessor).atZone(ZoneId.systemDefault()).toEpochSecond() * 1000L));
+ ZonedDateTime tz = ((LocalDateTime) temporalAccessor).atZone(ZoneId.systemDefault());
+ Timestamp timestamp = new Timestamp(tz.toEpochSecond() * 1000L);
+ timestamp.setNanos(tz.getNano());
+ vector.set(row, timestamp);
} else {
vect.noNulls = false;
vect.isNull[row] = true;
diff --git a/java/tools/src/test/org/apache/orc/tools/convert/TestCsvReader.java b/java/tools/src/test/org/apache/orc/tools/convert/TestCsvReader.java
index 6752de6..06ed5d2 100644
--- a/java/tools/src/test/org/apache/orc/tools/convert/TestCsvReader.java
+++ b/java/tools/src/test/org/apache/orc/tools/convert/TestCsvReader.java
@@ -183,10 +183,11 @@
@Test
public void testCustomTimestampFormat() throws Exception {
- String tsFormat = "d[d] MMM yyyy HH:mm:ss";
+ String tsFormat = "d[d] MMM yyyy HH:mm:ss.SSSSSS";
+
StringReader input = new StringReader(
- "'21 Mar 2018 12:23:34'\n" +
- "'3 Feb 2018 18:04:51'\n"
+ "'21 Mar 2018 12:23:34.123456'\n" +
+ "'3 Feb 2018 18:04:51.456789'\n"
);
TypeDescription schema = TypeDescription.fromString(
"struct<a:timestamp>");
@@ -196,7 +197,7 @@
assertEquals(true, reader.nextBatch(batch));
assertEquals(2, batch.size);
TimestampColumnVector cv = (TimestampColumnVector) batch.cols[0];
- assertEquals("2018-03-21 12:23:34.0", cv.asScratchTimestamp(0).toString());
- assertEquals("2018-02-03 18:04:51.0", cv.asScratchTimestamp(1).toString());
+ assertEquals("2018-03-21 12:23:34.123456", cv.asScratchTimestamp(0).toString());
+ assertEquals("2018-02-03 18:04:51.456789", cv.asScratchTimestamp(1).toString());
}
}
diff --git a/java/tools/src/test/org/apache/orc/tools/convert/TestJsonReader.java b/java/tools/src/test/org/apache/orc/tools/convert/TestJsonReader.java
new file mode 100644
index 0000000..10529d8
--- /dev/null
+++ b/java/tools/src/test/org/apache/orc/tools/convert/TestJsonReader.java
@@ -0,0 +1,30 @@
+package org.apache.orc.tools.convert;
+
+import org.apache.hadoop.hive.ql.exec.vector.TimestampColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
+import org.apache.orc.TypeDescription;
+import org.junit.Test;
+
+import java.io.StringReader;
+
+import static org.junit.Assert.assertEquals;
+
+public class TestJsonReader {
+ @Test
+ public void testCustomTimestampFormat() throws Exception {
+ String tsFormat = "yyyy-MM-dd HH:mm:ss.SSSSSS";
+
+ String s = "{\"a\":\"2018-03-21 12:23:34.123456\"}\n" +
+ "{\"a\":\"2018-02-03 18:04:51.456789\"}\n";
+ StringReader input = new StringReader(s);
+ TypeDescription schema = TypeDescription.fromString(
+ "struct<a:timestamp>");
+ JsonReader reader = new JsonReader(input, null, 1, schema, tsFormat);
+ VectorizedRowBatch batch = schema.createRowBatch(2);
+ assertEquals(true, reader.nextBatch(batch));
+ assertEquals(2, batch.size);
+ TimestampColumnVector cv = (TimestampColumnVector) batch.cols[0];
+ assertEquals("2018-03-21 12:23:34.123456", cv.asScratchTimestamp(0).toString());
+ assertEquals("2018-02-03 18:04:51.456789", cv.asScratchTimestamp(1).toString());
+ }
+}