ORC-578: Fix schema conversions with proleptic dates/times.
Fixes #462
Signed-off-by: Owen O'Malley <omalley@apache.org>
diff --git a/LICENSE b/LICENSE
index 1cb1082..7050787 100644
--- a/LICENSE
+++ b/LICENSE
@@ -331,3 +331,37 @@
If you redistribute modified sources, we would appreciate that you include in
the file ChangeLog history information documenting your changes. Please read
the FAQ for more information on the distribution of modified source versions.
+
+For orc.threeten:
+
+ /*
+ * Copyright (c) 2007-present, Stephen Colebourne & Michael Nascimento Santos
+ *
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ *
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ *
+ * * Neither the name of JSR-310 nor the names of its contributors
+ * may be used to endorse or promote products derived from this software
+ * without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+ * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+ * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
\ No newline at end of file
diff --git a/java/core/pom.xml b/java/core/pom.xml
index 194fc28..bfe9842 100644
--- a/java/core/pom.xml
+++ b/java/core/pom.xml
@@ -75,6 +75,10 @@
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.threeten</groupId>
+ <artifactId>threeten-extra</artifactId>
+ </dependency>
<!-- test inter-project -->
<dependency>
diff --git a/java/core/src/java/org/apache/orc/impl/ColumnStatisticsImpl.java b/java/core/src/java/org/apache/orc/impl/ColumnStatisticsImpl.java
index ad58bdd..f671b5f 100644
--- a/java/core/src/java/org/apache/orc/impl/ColumnStatisticsImpl.java
+++ b/java/core/src/java/org/apache/orc/impl/ColumnStatisticsImpl.java
@@ -1654,21 +1654,21 @@
maximum = DateUtils.convertTime(
SerializationUtils.convertToUtc(TimeZone.getDefault(),
timestampStats.getMaximum()),
- writerUsedProlepticGregorian, convertToProlepticGregorian);
+ writerUsedProlepticGregorian, convertToProlepticGregorian, true);
}
if (timestampStats.hasMinimum()) {
minimum = DateUtils.convertTime(
SerializationUtils.convertToUtc(TimeZone.getDefault(),
timestampStats.getMinimum()),
- writerUsedProlepticGregorian, convertToProlepticGregorian);
+ writerUsedProlepticGregorian, convertToProlepticGregorian, true);
}
if (timestampStats.hasMaximumUtc()) {
maximum = DateUtils.convertTime(timestampStats.getMaximumUtc(),
- writerUsedProlepticGregorian, convertToProlepticGregorian);
+ writerUsedProlepticGregorian, convertToProlepticGregorian, true);
}
if (timestampStats.hasMinimumUtc()) {
minimum = DateUtils.convertTime(timestampStats.getMinimumUtc(),
- writerUsedProlepticGregorian, convertToProlepticGregorian);
+ writerUsedProlepticGregorian, convertToProlepticGregorian, true);
}
}
diff --git a/java/core/src/java/org/apache/orc/impl/ConvertTreeReaderFactory.java b/java/core/src/java/org/apache/orc/impl/ConvertTreeReaderFactory.java
index 1ea870a..81a9b62 100644
--- a/java/core/src/java/org/apache/orc/impl/ConvertTreeReaderFactory.java
+++ b/java/core/src/java/org/apache/orc/impl/ConvertTreeReaderFactory.java
@@ -19,19 +19,24 @@
import java.io.IOException;
import java.nio.charset.StandardCharsets;
-import java.sql.Date;
+import java.sql.Timestamp;
import java.time.Instant;
import java.time.LocalDate;
import java.time.ZoneId;
+import java.time.chrono.Chronology;
+import java.time.chrono.IsoChronology;
import java.time.format.DateTimeFormatter;
import java.time.format.DateTimeFormatterBuilder;
import java.time.format.DateTimeParseException;
+import java.time.format.SignStyle;
+import java.time.temporal.ChronoField;
import java.util.EnumMap;
import java.util.TimeZone;
import org.apache.hadoop.hive.common.type.HiveDecimal;
import org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector;
import org.apache.hadoop.hive.ql.exec.vector.ColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.DateColumnVector;
import org.apache.hadoop.hive.ql.exec.vector.Decimal64ColumnVector;
import org.apache.hadoop.hive.ql.exec.vector.DecimalColumnVector;
import org.apache.hadoop.hive.ql.exec.vector.DoubleColumnVector;
@@ -39,12 +44,12 @@
import org.apache.hadoop.hive.ql.exec.vector.TimestampColumnVector;
import org.apache.hadoop.hive.ql.exec.vector.expressions.StringExpr;
import org.apache.hadoop.hive.ql.util.TimestampUtils;
-import org.apache.hadoop.hive.serde2.io.DateWritable;
import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable;
import org.apache.orc.OrcProto;
import org.apache.orc.TypeDescription;
import org.apache.orc.TypeDescription.Category;
import org.apache.orc.impl.reader.StripePlanner;
+import org.threeten.extra.chrono.HybridChronology;
/**
* Convert ORC tree readers.
@@ -596,9 +601,8 @@
@Override
public void setConvertVectorElement(int elementNum) {
- // Use TimestampWritable's getSeconds.
- long longValue = TimestampUtils.millisToSeconds(
- timestampColVector.asScratchTimestamp(elementNum).getTime());
+ long millis = timestampColVector.asScratchTimestamp(elementNum).getTime();
+ long longValue = Math.floorDiv(millis, 1000);
downCastAnyInteger(longColVector, elementNum, longValue, readerType);
}
@@ -739,8 +743,13 @@
@Override
public void setConvertVectorElement(int elementNum) throws IOException {
- doubleColVector.vector[elementNum] = TimestampUtils.getDouble(
- timestampColVector.asScratchTimestamp(elementNum));
+ Timestamp ts = timestampColVector.asScratchTimestamp(elementNum);
+ double result = Math.floorDiv(ts.getTime(), 1000);
+ int nano = ts.getNanos();
+ if (nano != 0) {
+ result += nano / 1_000_000_000.0;
+ }
+ doubleColVector.vector[elementNum] = result;
}
@Override
@@ -915,8 +924,12 @@
@Override
public void setConvertVectorElement(int elementNum) throws IOException {
- long seconds = timestampColVector.time[elementNum] / 1000;
+ long seconds = Math.floorDiv(timestampColVector.time[elementNum], 1000);
long nanos = timestampColVector.nanos[elementNum];
+ if (seconds < 0 && nanos > 0) {
+ seconds += 1;
+ nanos = 1_000_000_000 - nanos;
+ }
HiveDecimal value = HiveDecimal.create(String.format("%d.%09d", seconds, nanos));
if (value != null) {
// The DecimalColumnVector will enforce precision and scale and set the entry to null when out of bounds.
@@ -1131,7 +1144,12 @@
* Eg. "2019-07-09"
*/
static final DateTimeFormatter DATE_FORMAT =
- new DateTimeFormatterBuilder().appendPattern("uuuu-MM-dd")
+ new DateTimeFormatterBuilder()
+ .appendValue(ChronoField.YEAR, 4, 10, SignStyle.EXCEEDS_PAD)
+ .appendLiteral('-')
+ .appendValue(ChronoField.MONTH_OF_YEAR, 2)
+ .appendLiteral('-')
+ .appendValue(ChronoField.DAY_OF_MONTH, 2)
.toFormatter();
/**
@@ -1139,10 +1157,18 @@
* Eg. "2019-07-09 13:11:00"
*/
static final DateTimeFormatter TIMESTAMP_FORMAT =
- new DateTimeFormatterBuilder()
- .append(DATE_FORMAT)
- .appendPattern(" HH:mm:ss[.S]")
- .toFormatter();
+ new DateTimeFormatterBuilder()
+ .append(DATE_FORMAT)
+ .appendLiteral(' ')
+ .appendValue(ChronoField.HOUR_OF_DAY, 2)
+ .appendLiteral(':')
+ .appendValue(ChronoField.MINUTE_OF_HOUR, 2)
+ .optionalStart()
+ .appendLiteral(':')
+ .appendValue(ChronoField.SECOND_OF_MINUTE, 2)
+ .optionalStart()
+ .appendFraction(ChronoField.NANO_OF_SECOND, 0, 9, true)
+ .toFormatter();
/**
* The format for converting from/to string/timestamp with local time zone.
@@ -1154,6 +1180,9 @@
.appendPattern(" VV")
.toFormatter();
+ static final long MIN_EPOCH_SECONDS = Instant.MIN.getEpochSecond();
+ static final long MAX_EPOCH_SECONDS = Instant.MAX.getEpochSecond();
+
/**
* Create an Instant from an entry in a TimestampColumnVector.
* It assumes that vector.isRepeating and null values have been handled
@@ -1163,7 +1192,7 @@
* @return a timestamp Instant
*/
static Instant timestampToInstant(TimestampColumnVector vector, int element) {
- return Instant.ofEpochSecond(vector.time[element] / 1000,
+ return Instant.ofEpochSecond(Math.floorDiv(vector.time[element], 1000),
vector.nanos[element]);
}
@@ -1177,10 +1206,14 @@
// copy the value so that we can mutate it
HiveDecimalWritable value = new HiveDecimalWritable(vector.vector[element]);
long seconds = value.longValue();
- value.mutateFractionPortion();
- value.mutateScaleByPowerOfTen(9);
- int nanos = (int) value.longValue();
- return Instant.ofEpochSecond(seconds, nanos);
+ if (seconds < MIN_EPOCH_SECONDS || seconds > MAX_EPOCH_SECONDS) {
+ return null;
+ } else {
+ value.mutateFractionPortion();
+ value.mutateScaleByPowerOfTen(9);
+ int nanos = (int) value.longValue();
+ return Instant.ofEpochSecond(seconds, nanos);
+ }
}
public static class StringGroupFromTimestampTreeReader extends ConvertTreeReader {
@@ -1197,7 +1230,10 @@
this.readerType = readerType;
local = context.getUseUTCTimestamp() ? ZoneId.of("UTC")
: ZoneId.systemDefault();
- formatter = instantType ? INSTANT_TIMESTAMP_FORMAT : TIMESTAMP_FORMAT;
+ Chronology chronology = context.useProlepticGregorian()
+ ? IsoChronology.INSTANCE : HybridChronology.INSTANCE;
+ formatter = (instantType ? INSTANT_TIMESTAMP_FORMAT : TIMESTAMP_FORMAT)
+ .withChronology(chronology);
}
@Override
@@ -1226,22 +1262,22 @@
public static class StringGroupFromDateTreeReader extends ConvertTreeReader {
private final TypeDescription readerType;
- private LongColumnVector longColVector;
+ private DateColumnVector longColVector;
private BytesColumnVector bytesColVector;
- private Date date;
+ private final boolean useProlepticGregorian;
StringGroupFromDateTreeReader(int columnId, TypeDescription readerType,
Context context) throws IOException {
super(columnId, new DateTreeReader(columnId, context));
this.readerType = readerType;
- date = new Date(0);
+ useProlepticGregorian = context.useProlepticGregorian();
}
@Override
- public void setConvertVectorElement(int elementNum) throws IOException {
- date.setTime(DateWritable.daysToMillis((int) longColVector.vector[elementNum]));
- String string = date.toString();
- byte[] bytes = string.getBytes(StandardCharsets.UTF_8);
+ public void setConvertVectorElement(int elementNum) {
+ String dateStr = DateUtils.printDate((int) (longColVector.vector[elementNum]),
+ useProlepticGregorian);
+ byte[] bytes = dateStr.getBytes(StandardCharsets.UTF_8);
assignStringGroupVectorEntry(bytesColVector, elementNum, readerType, bytes);
}
@@ -1251,7 +1287,7 @@
final int batchSize) throws IOException {
if (longColVector == null) {
// Allocate column vector for file; cast column vector for reader.
- longColVector = new LongColumnVector();
+ longColVector = new DateColumnVector();
bytesColVector = (BytesColumnVector) previousVector;
}
// Read present/isNull stream
@@ -1352,6 +1388,8 @@
private TimestampColumnVector timestampColVector;
private final boolean useUtc;
private final TimeZone local;
+ private final boolean fileUsedProlepticGregorian;
+ private final boolean useProlepticGregorian;
TimestampFromAnyIntegerTreeReader(int columnId, TypeDescription fileType,
Context context,
@@ -1359,6 +1397,8 @@
super(columnId, createFromInteger(columnId, fileType, context));
this.useUtc = isInstant || context.getUseUTCTimestamp();
local = TimeZone.getDefault();
+ fileUsedProlepticGregorian = context.fileUsedProlepticGregorian();
+ useProlepticGregorian = context.useProlepticGregorian();
}
@Override
@@ -1379,10 +1419,12 @@
longColVector = new LongColumnVector();
timestampColVector = (TimestampColumnVector) previousVector;
}
+ timestampColVector.changeCalendar(fileUsedProlepticGregorian, false);
// Read present/isNull stream
fromReader.nextVector(longColVector, isNull, batchSize);
convertVector(longColVector, timestampColVector, batchSize);
+ timestampColVector.changeCalendar(useProlepticGregorian, true);
}
}
@@ -1391,6 +1433,8 @@
private TimestampColumnVector timestampColVector;
private final boolean useUtc;
private final TimeZone local;
+ private final boolean useProlepticGregorian;
+ private final boolean fileUsedProlepticGregorian;
TimestampFromDoubleTreeReader(int columnId, TypeDescription fileType,
TypeDescription readerType, Context context) throws IOException {
@@ -1400,6 +1444,8 @@
useUtc = readerType.getCategory() == Category.TIMESTAMP_INSTANT ||
context.getUseUTCTimestamp();
local = TimeZone.getDefault();
+ useProlepticGregorian = context.useProlepticGregorian();
+ fileUsedProlepticGregorian = context.fileUsedProlepticGregorian();
}
@Override
@@ -1408,11 +1454,9 @@
if (!useUtc) {
seconds = SerializationUtils.convertFromUtc(local, seconds);
}
- long wholeSec = (long) Math.floor(seconds);
-
// overflow
double doubleMillis = seconds * 1000;
- long millis = wholeSec * 1000;
+ long millis = Math.round(doubleMillis);
if (doubleMillis > Long.MAX_VALUE || doubleMillis < Long.MIN_VALUE ||
((millis >= 0) != (doubleMillis >= 0))) {
timestampColVector.time[elementNum] = 0L;
@@ -1420,9 +1464,9 @@
timestampColVector.isNull[elementNum] = true;
timestampColVector.noNulls = false;
} else {
- timestampColVector.time[elementNum] = wholeSec * 1000;
+ timestampColVector.time[elementNum] = millis;
timestampColVector.nanos[elementNum] =
- 1_000_000 * (int) Math.round((seconds - wholeSec) * 1000);
+ (int) Math.floorMod(millis, 1000) * 1_000_000;
}
}
@@ -1435,10 +1479,12 @@
doubleColVector = new DoubleColumnVector();
timestampColVector = (TimestampColumnVector) previousVector;
}
+ timestampColVector.changeCalendar(fileUsedProlepticGregorian, false);
// Read present/isNull stream
fromReader.nextVector(doubleColVector, isNull, batchSize);
convertVector(doubleColVector, timestampColVector, batchSize);
+ timestampColVector.changeCalendar(useProlepticGregorian, true);
}
}
@@ -1449,6 +1495,8 @@
private TimestampColumnVector timestampColVector;
private final boolean useUtc;
private final TimeZone local;
+ private final boolean useProlepticGregorian;
+ private final boolean fileUsedProlepticGregorian;
TimestampFromDecimalTreeReader(int columnId, TypeDescription fileType,
Context context,
@@ -1459,17 +1507,23 @@
this.scale = fileType.getScale();
useUtc = isInstant || context.getUseUTCTimestamp();
local = TimeZone.getDefault();
+ useProlepticGregorian = context.useProlepticGregorian();
+ fileUsedProlepticGregorian = context.fileUsedProlepticGregorian();
}
@Override
public void setConvertVectorElement(int elementNum) {
Instant t = decimalToInstant(decimalColVector, elementNum);
- if (!useUtc) {
+ if (t == null) {
+ timestampColVector.noNulls = false;
+ timestampColVector.isNull[elementNum] = true;
+ } else if (!useUtc) {
+ long millis = t.toEpochMilli();
timestampColVector.time[elementNum] =
- SerializationUtils.convertFromUtc(local, t.getEpochSecond() * 1000);
+ SerializationUtils.convertFromUtc(local, millis);
timestampColVector.nanos[elementNum] = t.getNano();
} else {
- timestampColVector.time[elementNum] = t.getEpochSecond() * 1000;
+ timestampColVector.time[elementNum] = t.toEpochMilli();
timestampColVector.nanos[elementNum] = t.getNano();
}
}
@@ -1483,10 +1537,12 @@
decimalColVector = new DecimalColumnVector(precision, scale);
timestampColVector = (TimestampColumnVector) previousVector;
}
+ timestampColVector.changeCalendar(fileUsedProlepticGregorian, false);
// Read present/isNull stream
fromReader.nextVector(decimalColVector, isNull, batchSize);
convertVector(decimalColVector, timestampColVector, batchSize);
+ timestampColVector.changeCalendar(useProlepticGregorian, true);
}
}
@@ -1494,17 +1550,24 @@
private BytesColumnVector bytesColVector;
private TimestampColumnVector timestampColVector;
private final DateTimeFormatter formatter;
+ private final boolean useProlepticGregorian;
TimestampFromStringGroupTreeReader(int columnId, TypeDescription fileType,
Context context, boolean isInstant)
throws IOException {
super(columnId, getStringGroupTreeReader(columnId, fileType, context));
+ useProlepticGregorian = context.useProlepticGregorian();
+ Chronology chronology = useProlepticGregorian
+ ? IsoChronology.INSTANCE
+ : HybridChronology.INSTANCE;
if (isInstant) {
- formatter = INSTANT_TIMESTAMP_FORMAT;
+ formatter = INSTANT_TIMESTAMP_FORMAT.withChronology(chronology);
} else {
- formatter = TIMESTAMP_FORMAT.withZone(context.getUseUTCTimestamp() ?
- ZoneId.of("UTC") :
- ZoneId.systemDefault());
+ formatter = TIMESTAMP_FORMAT
+ .withZone(context.getUseUTCTimestamp() ?
+ ZoneId.of("UTC") :
+ ZoneId.systemDefault())
+ .withChronology(chronology);
}
}
@@ -1514,9 +1577,9 @@
elementNum);
try {
Instant instant = Instant.from(formatter.parse(str));
- timestampColVector.time[elementNum] = instant.getEpochSecond() * 1000;
+ timestampColVector.time[elementNum] = instant.toEpochMilli();
timestampColVector.nanos[elementNum] = instant.getNano();
- } catch (DateTimeParseException exception) {
+ } catch (DateTimeParseException e) {
timestampColVector.noNulls = false;
timestampColVector.isNull[elementNum] = true;
}
@@ -1535,20 +1598,23 @@
fromReader.nextVector(bytesColVector, isNull, batchSize);
convertVector(bytesColVector, timestampColVector, batchSize);
+ timestampColVector.changeCalendar(useProlepticGregorian, false);
}
}
public static class TimestampFromDateTreeReader extends ConvertTreeReader {
- private LongColumnVector longColVector;
+ private DateColumnVector longColVector;
private TimestampColumnVector timestampColVector;
private final boolean useUtc;
private final TimeZone local = TimeZone.getDefault();
+ private final boolean useProlepticGregorian;
TimestampFromDateTreeReader(int columnId, TypeDescription readerType,
Context context) throws IOException {
super(columnId, new DateTreeReader(columnId, context));
useUtc = readerType.getCategory() == Category.TIMESTAMP_INSTANT ||
context.getUseUTCTimestamp();
+ useProlepticGregorian = context.useProlepticGregorian();
}
@Override
@@ -1567,32 +1633,36 @@
final int batchSize) throws IOException {
if (longColVector == null) {
// Allocate column vector for file; cast column vector for reader.
- longColVector = new LongColumnVector();
+ longColVector = new DateColumnVector();
timestampColVector = (TimestampColumnVector) previousVector;
}
// Read present/isNull stream
fromReader.nextVector(longColVector, isNull, batchSize);
convertVector(longColVector, timestampColVector, batchSize);
+ timestampColVector.changeCalendar(useProlepticGregorian, false);
}
}
public static class DateFromStringGroupTreeReader extends ConvertTreeReader {
private BytesColumnVector bytesColVector;
private LongColumnVector longColVector;
+ private DateColumnVector dateColumnVector;
+ private final boolean useProlepticGregorian;
DateFromStringGroupTreeReader(int columnId, TypeDescription fileType, Context context)
throws IOException {
super(columnId, getStringGroupTreeReader(columnId, fileType, context));
+ useProlepticGregorian = context.useProlepticGregorian();
}
@Override
public void setConvertVectorElement(int elementNum) {
String stringValue =
SerializationUtils.bytesVectorToString(bytesColVector, elementNum);
- Date dateValue = SerializationUtils.parseDateFromString(stringValue);
+ Integer dateValue = DateUtils.parseDate(stringValue, useProlepticGregorian);
if (dateValue != null) {
- longColVector.vector[elementNum] = DateWritable.dateToDays(dateValue);
+ longColVector.vector[elementNum] = dateValue;
} else {
longColVector.noNulls = false;
longColVector.isNull[elementNum] = true;
@@ -1607,11 +1677,23 @@
// Allocate column vector for file; cast column vector for reader.
bytesColVector = new BytesColumnVector();
longColVector = (LongColumnVector) previousVector;
+ if (longColVector instanceof DateColumnVector) {
+ dateColumnVector = (DateColumnVector) longColVector;
+ } else {
+ dateColumnVector = null;
+ if (useProlepticGregorian) {
+ throw new IllegalArgumentException("Can't use LongColumnVector with" +
+ " proleptic Gregorian dates.");
+ }
+ }
}
// Read present/isNull stream
fromReader.nextVector(bytesColVector, isNull, batchSize);
convertVector(bytesColVector, longColVector, batchSize);
+ if (dateColumnVector != null) {
+ dateColumnVector.changeCalendar(useProlepticGregorian, false);
+ }
}
}
@@ -1619,12 +1701,14 @@
private TimestampColumnVector timestampColVector;
private LongColumnVector longColVector;
private final ZoneId local;
+ private final boolean useProlepticGregorian;
DateFromTimestampTreeReader(int columnId, Context context,
boolean instantType) throws IOException {
super(columnId, new TimestampTreeReader(columnId, context, instantType));
boolean useUtc = instantType || context.getUseUTCTimestamp();
local = useUtc ? ZoneId.of("UTC") : ZoneId.systemDefault();
+ useProlepticGregorian = context.useProlepticGregorian();
}
@Override
@@ -1644,11 +1728,19 @@
// Allocate column vector for file; cast column vector for reader.
timestampColVector = new TimestampColumnVector();
longColVector = (LongColumnVector) previousVector;
+ if (useProlepticGregorian && !(longColVector instanceof DateColumnVector)) {
+ throw new IllegalArgumentException("Can't use LongColumnVector with" +
+ " proleptic Gregorian dates.");
+ }
}
// Read present/isNull stream
fromReader.nextVector(timestampColVector, isNull, batchSize);
convertVector(timestampColVector, longColVector, batchSize);
+ if (longColVector instanceof DateColumnVector) {
+ ((DateColumnVector) longColVector)
+ .changeCalendar(useProlepticGregorian, false);
+ }
}
}
diff --git a/java/core/src/java/org/apache/orc/impl/DateUtils.java b/java/core/src/java/org/apache/orc/impl/DateUtils.java
index 8ac574c..0b7592c 100644
--- a/java/core/src/java/org/apache/orc/impl/DateUtils.java
+++ b/java/core/src/java/org/apache/orc/impl/DateUtils.java
@@ -17,11 +17,15 @@
*/
package org.apache.orc.impl;
-import java.text.ParseException;
-import java.text.SimpleDateFormat;
-import java.util.Date;
-import java.util.GregorianCalendar;
-import java.util.TimeZone;
+import org.threeten.extra.chrono.HybridChronology;
+
+import java.time.Instant;
+import java.time.LocalDate;
+import java.time.ZoneId;
+import java.time.chrono.IsoChronology;
+import java.time.format.DateTimeFormatter;
+import java.time.format.DateTimeParseException;
+import java.time.temporal.TemporalAccessor;
import java.util.concurrent.TimeUnit;
/**
@@ -36,41 +40,39 @@
* old dates.
*/
public class DateUtils {
- private static SimpleDateFormat createFormatter(String fmt,
- GregorianCalendar calendar) {
- SimpleDateFormat result = new SimpleDateFormat(fmt);
- result.setCalendar(calendar);
- return result;
- }
-
- private static final String DATE = "yyyy-MM-dd";
- private static final String TIME = DATE + " HH:mm:ss";
- private static final TimeZone UTC = TimeZone.getTimeZone("UTC");
- private static final GregorianCalendar HYBRID = new GregorianCalendar();
- private static final ThreadLocal<SimpleDateFormat> HYBRID_DATE_FORMAT =
- ThreadLocal.withInitial(() -> createFormatter(DATE, HYBRID));
- private static final ThreadLocal<SimpleDateFormat> HYBRID_TIME_FORMAT =
- ThreadLocal.withInitial(() -> createFormatter(TIME, HYBRID));
+ private static final ZoneId UTC = ZoneId.of("UTC");
+ private static final ZoneId LOCAL = ZoneId.systemDefault();
private static final long SWITCHOVER_MILLIS;
private static final long SWITCHOVER_DAYS;
- private static final GregorianCalendar PROLEPTIC = new GregorianCalendar();
- private static final ThreadLocal<SimpleDateFormat> PROLEPTIC_DATE_FORMAT =
- ThreadLocal.withInitial(() -> createFormatter(DATE, PROLEPTIC));
- private static final ThreadLocal<SimpleDateFormat> PROLEPTIC_TIME_FORMAT =
- ThreadLocal.withInitial(() -> createFormatter(TIME, PROLEPTIC));
+ private static final DateTimeFormatter HYBRID_DATE_FORMAT =
+ ConvertTreeReaderFactory.DATE_FORMAT
+ .withChronology(HybridChronology.INSTANCE)
+ .withZone(UTC);
+ private static final DateTimeFormatter PROLEPTIC_DATE_FORMAT =
+ DateTimeFormatter.ISO_LOCAL_DATE
+ .withChronology(IsoChronology.INSTANCE)
+ .withZone(UTC);
+ private static final DateTimeFormatter HYBRID_UTC_TIME_FORMAT =
+ ConvertTreeReaderFactory.TIMESTAMP_FORMAT
+ .withChronology(HybridChronology.INSTANCE)
+ .withZone(UTC);
+ private static final DateTimeFormatter HYBRID_LOCAL_TIME_FORMAT =
+ ConvertTreeReaderFactory.TIMESTAMP_FORMAT
+ .withChronology(HybridChronology.INSTANCE)
+ .withZone(LOCAL);
+ private static final DateTimeFormatter PROLEPTIC_UTC_TIME_FORMAT =
+ ConvertTreeReaderFactory.TIMESTAMP_FORMAT
+ .withChronology(IsoChronology.INSTANCE)
+ .withZone(UTC);
+ private static final DateTimeFormatter PROLEPTIC_LOCAL_TIME_FORMAT =
+ ConvertTreeReaderFactory.TIMESTAMP_FORMAT
+ .withChronology(IsoChronology.INSTANCE)
+ .withZone(LOCAL);
static {
- HYBRID.setTimeZone(UTC);
- PROLEPTIC.setTimeZone(UTC);
- PROLEPTIC.setGregorianChange(new Date(Long.MIN_VALUE));
-
// Get the last day where the two calendars agree with each other.
- try {
- SWITCHOVER_MILLIS = HYBRID_DATE_FORMAT.get().parse("1582-10-15").getTime();
- SWITCHOVER_DAYS = TimeUnit.MILLISECONDS.toDays(SWITCHOVER_MILLIS);
- } catch (ParseException e) {
- throw new IllegalArgumentException("Can't parse switch over date", e);
- }
+ SWITCHOVER_DAYS = LocalDate.from(HYBRID_DATE_FORMAT.parse("1582-10-15")).toEpochDay();
+ SWITCHOVER_MILLIS = TimeUnit.DAYS.toMillis(SWITCHOVER_DAYS);
}
/**
@@ -82,14 +84,8 @@
public static int convertDateToProleptic(int hybrid) {
int proleptic = hybrid;
if (hybrid < SWITCHOVER_DAYS) {
- String dateStr = HYBRID_DATE_FORMAT.get().format(
- new Date(TimeUnit.DAYS.toMillis(hybrid)));
- try {
- proleptic = (int) TimeUnit.MILLISECONDS.toDays(
- PROLEPTIC_DATE_FORMAT.get().parse(dateStr).getTime());
- } catch (ParseException e) {
- throw new IllegalArgumentException("Can't parse " + dateStr, e);
- }
+ String dateStr = HYBRID_DATE_FORMAT.format(LocalDate.ofEpochDay(proleptic));
+ proleptic = (int) LocalDate.from(PROLEPTIC_DATE_FORMAT.parse(dateStr)).toEpochDay();
}
return proleptic;
}
@@ -103,18 +99,55 @@
public static int convertDateToHybrid(int proleptic) {
int hyrbid = proleptic;
if (proleptic < SWITCHOVER_DAYS) {
- String dateStr = PROLEPTIC_DATE_FORMAT.get().format(
- new Date(TimeUnit.DAYS.toMillis(proleptic)));
- try {
- hyrbid = (int) TimeUnit.MILLISECONDS.toDays(
- HYBRID_DATE_FORMAT.get().parse(dateStr).getTime());
- } catch (ParseException e) {
- throw new IllegalArgumentException("Can't parse " + dateStr, e);
- }
+ String dateStr = PROLEPTIC_DATE_FORMAT.format(LocalDate.ofEpochDay(proleptic));
+ hyrbid = (int) LocalDate.from(HYBRID_DATE_FORMAT.parse(dateStr)).toEpochDay();
}
return hyrbid;
}
+ /**
+ * Convert epoch millis from the hybrid Julian/Gregorian calendar to the
+ * proleptic Gregorian.
+ * @param hybrid millis of epoch in the hybrid Julian/Gregorian
+ * @param useUtc use UTC instead of local
+ * @return millis of epoch in the proleptic Gregorian
+ */
+ public static long convertTimeToProleptic(long hybrid, boolean useUtc) {
+ long proleptic = hybrid;
+ if (hybrid < SWITCHOVER_MILLIS) {
+ if (useUtc) {
+ String dateStr = HYBRID_UTC_TIME_FORMAT.format(Instant.ofEpochMilli(hybrid));
+ proleptic = Instant.from(PROLEPTIC_UTC_TIME_FORMAT.parse(dateStr)).toEpochMilli();
+ } else {
+ String dateStr = HYBRID_LOCAL_TIME_FORMAT.format(Instant.ofEpochMilli(hybrid));
+ proleptic = Instant.from(PROLEPTIC_LOCAL_TIME_FORMAT.parse(dateStr)).toEpochMilli();
+ }
+ }
+ return proleptic;
+ }
+
+ /**
+ * Convert epoch millis from the proleptic Gregorian calendar to the hybrid
+ * Julian/Gregorian.
+ * @param proleptic millis of epoch in the proleptic Gregorian
+ * @param useUtc use UTC instead of local
+ * @return millis of epoch in the hybrid Julian/Gregorian
+ */
+ public static long convertTimeToHybrid(long proleptic, boolean useUtc) {
+ long hybrid = proleptic;
+ if (proleptic < SWITCHOVER_MILLIS) {
+ if (useUtc) {
+ String dateStr = PROLEPTIC_UTC_TIME_FORMAT.format(Instant.ofEpochMilli(hybrid));
+ hybrid = Instant.from(HYBRID_UTC_TIME_FORMAT.parse(dateStr)).toEpochMilli();
+ } else {
+ String dateStr = PROLEPTIC_LOCAL_TIME_FORMAT.format(Instant.ofEpochMilli(hybrid));
+ hybrid = Instant.from(HYBRID_LOCAL_TIME_FORMAT.parse(dateStr)).toEpochMilli();
+ }
+ }
+ return hybrid;
+ }
+
+
public static int convertDate(int original,
boolean fromProleptic,
boolean toProleptic) {
@@ -129,51 +162,52 @@
public static long convertTime(long original,
boolean fromProleptic,
- boolean toProleptic) {
+ boolean toProleptic,
+ boolean useUtc) {
if (fromProleptic != toProleptic) {
return toProleptic
- ? convertTimeToProleptic(original)
- : convertTimeToHybrid(original);
+ ? convertTimeToProleptic(original, useUtc)
+ : convertTimeToHybrid(original, useUtc);
} else {
return original;
}
}
- /**
- * Convert epoch millis from the hybrid Julian/Gregorian calendar to the
- * proleptic Gregorian.
- * @param hybrid millis of epoch in the hybrid Julian/Gregorian
- * @return millis of epoch in the proleptic Gregorian
- */
- public static long convertTimeToProleptic(long hybrid) {
- long proleptic = hybrid;
- if (hybrid < SWITCHOVER_MILLIS) {
- String dateStr = HYBRID_TIME_FORMAT.get().format(new Date(hybrid));
- try {
- proleptic = PROLEPTIC_TIME_FORMAT.get().parse(dateStr).getTime();
- } catch (ParseException e) {
- throw new IllegalArgumentException("Can't parse " + dateStr, e);
- }
+
+ public static Integer parseDate(String date, boolean fromProleptic) {
+ try {
+ TemporalAccessor time = (fromProleptic ? PROLEPTIC_DATE_FORMAT : HYBRID_DATE_FORMAT).parse(date);
+ return (int) LocalDate.from(time).toEpochDay();
+ } catch (DateTimeParseException e) {
+ return null;
}
- return proleptic;
}
- /**
- * Convert epoch millis from the proleptic Gregorian calendar to the hybrid
- * Julian/Gregorian.
- * @param proleptic millis of epoch in the proleptic Gregorian
- * @return millis of epoch in the hybrid Julian/Gregorian
- */
- public static long convertTimeToHybrid(long proleptic) {
- long hybrid = proleptic;
- if (proleptic < SWITCHOVER_MILLIS) {
- String dateStr = PROLEPTIC_TIME_FORMAT.get().format(new Date(proleptic));
- try {
- hybrid = HYBRID_TIME_FORMAT.get().parse(dateStr).getTime();
- } catch (ParseException e) {
- throw new IllegalArgumentException("Can't parse " + dateStr, e);
- }
+ public static String printDate(int date, boolean fromProleptic) {
+ return (fromProleptic ? PROLEPTIC_DATE_FORMAT : HYBRID_DATE_FORMAT)
+ .format(LocalDate.ofEpochDay(date));
+ }
+
+ public static DateTimeFormatter getTimeFormat(boolean useProleptic,
+ boolean useUtc) {
+ if (useProleptic) {
+ return useUtc ? PROLEPTIC_UTC_TIME_FORMAT : PROLEPTIC_LOCAL_TIME_FORMAT;
+ } else {
+ return useUtc ? HYBRID_UTC_TIME_FORMAT : HYBRID_LOCAL_TIME_FORMAT;
}
- return hybrid;
+ }
+
+ public static Long parseTime(String date, boolean fromProleptic, boolean useUtc) {
+ try {
+ TemporalAccessor time = getTimeFormat(fromProleptic, useUtc).parse(date);
+ return Instant.from(time).toEpochMilli();
+ } catch (DateTimeParseException e) {
+ return null;
+ }
+ }
+
+ public static String printTime(long millis, boolean fromProleptic,
+ boolean useUtc) {
+ return getTimeFormat(fromProleptic, useUtc).format(Instant.ofEpochMilli(millis));
}
private DateUtils() {
diff --git a/java/core/src/java/org/apache/orc/impl/WriterImpl.java b/java/core/src/java/org/apache/orc/impl/WriterImpl.java
index 3d9f380..c03e1b4 100644
--- a/java/core/src/java/org/apache/orc/impl/WriterImpl.java
+++ b/java/core/src/java/org/apache/orc/impl/WriterImpl.java
@@ -640,11 +640,9 @@
rawDataSize = computeRawDataSize();
// serialize the types
writeTypes(builder, schema);
- if (hasDateOrTime(schema)) {
- builder.setCalendar(useProlepticGregorian
+ builder.setCalendar(useProlepticGregorian
? OrcProto.CalendarKind.PROLEPTIC_GREGORIAN
: OrcProto.CalendarKind.JULIAN_GREGORIAN);
- }
// add the stripe information
for(OrcProto.StripeInformation stripe: stripes) {
builder.addStripes(stripe);
@@ -866,25 +864,6 @@
return false;
}
- private static boolean hasDateOrTime(TypeDescription schema) {
- switch (schema.getCategory()) {
- case TIMESTAMP:
- case TIMESTAMP_INSTANT:
- case DATE:
- return true;
- default:
- }
- List<TypeDescription> children = schema.getChildren();
- if (children != null) {
- for(TypeDescription child: children) {
- if (hasDateOrTime(child)) {
- return true;
- }
- }
- }
- return false;
- }
-
private WriterEncryptionKey getKey(String keyName,
KeyProvider provider) throws IOException {
WriterEncryptionKey result = keys.get(keyName);
diff --git a/java/core/src/java/org/threeten/extra/chrono/HybridChronology.java b/java/core/src/java/org/threeten/extra/chrono/HybridChronology.java
new file mode 100644
index 0000000..eee9fe6
--- /dev/null
+++ b/java/core/src/java/org/threeten/extra/chrono/HybridChronology.java
@@ -0,0 +1,455 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.threeten.extra.chrono;
+
+import java.io.Serializable;
+import java.time.Clock;
+import java.time.DateTimeException;
+import java.time.Instant;
+import java.time.LocalDate;
+import java.time.ZoneId;
+import java.time.chrono.AbstractChronology;
+import java.time.chrono.ChronoLocalDateTime;
+import java.time.chrono.ChronoZonedDateTime;
+import java.time.chrono.Chronology;
+import java.time.chrono.Era;
+import java.time.chrono.IsoChronology;
+import java.time.format.ResolverStyle;
+import java.time.temporal.ChronoField;
+import java.time.temporal.TemporalAccessor;
+import java.time.temporal.TemporalField;
+import java.time.temporal.ValueRange;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * The Julian-Gregorian hybrid calendar system.
+ * <p>
+ * The British calendar system follows the rules of the Julian calendar
+ * until 1752 and the rules of the Gregorian (ISO) calendar since then.
+ * The Julian differs from the Gregorian only in terms of the leap year rule.
+ * <p>
+ * The Julian and Gregorian calendar systems are linked to Rome and the Vatican
+ * with the Julian preceding the Gregorian. The Gregorian was introduced to
+ * handle the drift of the seasons through the year due to the inaccurate
+ * Julian leap year rules. When first introduced by the Vatican in 1582,
+ * the cutover resulted in a "gap" of 10 days.
+ * <p>
+ * While the calendar was introduced in 1582, it was not adopted everywhere.
+ * Britain did not adopt it until the 1752, when Wednesday 2nd September 1752
+ * was followed by Thursday 14th September 1752.
+ * <p>
+ * This chronology implements the proleptic Julian calendar system followed by
+ * the proleptic Gregorian calendar system (identical to the ISO calendar system).
+ * Dates are aligned such that {@code 0001-01-01 (British)} is {@code 0000-12-30 (ISO)}.
+ * <p>
+ * This class implements a calendar where January 1st is the start of the year.
+ * The history of the start of the year is complex and using the current standard
+ * is the most consistent.
+ * <p>
+ * The eras of this calendar system are defined by {@link JulianEra} to avoid unnecessary duplication.
+ * <p>
+ * The fields are defined as follows:
+ * <ul>
+ * <li>era - There are two eras, the current 'Anno Domini' (AD) and the previous era 'Before Christ' (BC).
+ * <li>year-of-era - The year-of-era for the current era increases uniformly from the epoch at year one.
+ * For the previous era the year increases from one as time goes backwards.
+ * <li>proleptic-year - The proleptic year is the same as the year-of-era for the
+ * current era. For the previous era, years have zero, then negative values.
+ * <li>month-of-year - There are 12 months in a year, numbered from 1 to 12.
+ * <li>day-of-month - There are between 28 and 31 days in each month, numbered from 1 to 31.
+ * Months 4, 6, 9 and 11 have 30 days, Months 1, 3, 5, 7, 8, 10 and 12 have 31 days.
+ * Month 2 has 28 days, or 29 in a leap year.
+ * The cutover month, September 1752, has a value range from 1 to 30, but a length of 19.
+ * <li>day-of-year - There are 365 days in a standard year and 366 in a leap year.
+ * The days are numbered from 1 to 365 or 1 to 366.
+ * The cutover year 1752 has values from 1 to 355 and a length of 355 days.
+ * </ul>
+ *
+ * <h3>Implementation Requirements</h3>
+ * This class is immutable and thread-safe.
+ */
+public final class HybridChronology
+ extends AbstractChronology
+ implements Serializable {
+
+ /**
+ * Singleton instance for the Coptic chronology.
+ */
+ public static final HybridChronology INSTANCE = new HybridChronology();
+ /**
+ * The cutover date, October 15, 1582.
+ */
+ public static final LocalDate CUTOVER = LocalDate.of(1582, 10, 15);
+ /**
+ * The number of cutover days.
+ */
+ static final int CUTOVER_DAYS = 10;
+ /**
+ * The cutover year.
+ */
+ static final int CUTOVER_YEAR = 1582;
+
+ /**
+ * Serialization version.
+ */
+ private static final long serialVersionUID = 87235724675472658L;
+ /**
+ * Range of day-of-year.
+ */
+ static final ValueRange DOY_RANGE = ValueRange.of(1, 355, 366);
+ /**
+ * Range of aligned-week-of-month.
+ */
+ static final ValueRange ALIGNED_WOM_RANGE = ValueRange.of(1, 3, 5);
+ /**
+ * Range of aligned-week-of-year.
+ */
+ static final ValueRange ALIGNED_WOY_RANGE = ValueRange.of(1, 51, 53);
+ /**
+ * Range of proleptic-year.
+ */
+ static final ValueRange YEAR_RANGE = ValueRange.of(-999_998, 999_999);
+ /**
+ * Range of year.
+ */
+ static final ValueRange YOE_RANGE = ValueRange.of(1, 999_999);
+ /**
+ * Range of proleptic month.
+ */
+ static final ValueRange PROLEPTIC_MONTH_RANGE = ValueRange.of(-999_998 * 12L, 999_999 * 12L + 11);
+
+ /**
+ * Private constructor, that is public to satisfy the {@code ServiceLoader}.
+ * @deprecated Use the singleton {@link #INSTANCE} instead.
+ */
+ @Deprecated
+ public HybridChronology() {
+ }
+
+ /**
+ * Resolve singleton.
+ *
+ * @return the singleton instance, not null
+ */
+ private Object readResolve() {
+ return INSTANCE;
+ }
+
+ //-------------------------------------------------------------------------
+ /**
+ * Gets the cutover date between the Julian and Gregorian calendar.
+ * <p>
+ * The date returned is the first date that the Gregorian (ISO) calendar applies,
+ * which is Thursday 14th September 1752.
+ *
+ * @return the first date after the cutover, not null
+ */
+ public LocalDate getCutover() {
+ return CUTOVER;
+ }
+
+ //-----------------------------------------------------------------------
+ /**
+ * Gets the ID of the chronology - 'Hybrid'.
+ * <p>
+ * The ID uniquely identifies the {@code Chronology}.
+ * It can be used to lookup the {@code Chronology} using {@link Chronology#of(String)}.
+ *
+ * @return the chronology ID - 'Hybrid'
+ * @see #getCalendarType()
+ */
+ @Override
+ public String getId() {
+ return "Hybrid";
+ }
+
+ /**
+ * Gets the calendar type of the underlying calendar system, which returns null.
+ * <p>
+ * The <em>Unicode Locale Data Markup Language (LDML)</em> specification
+ * does not define an identifier for this calendar system, thus null is returned.
+ *
+ * @return the calendar system type, null
+ * @see #getId()
+ */
+ @Override
+ public String getCalendarType() {
+ return null;
+ }
+
+ //-----------------------------------------------------------------------
+ /**
+ * Obtains a local date in British Cutover calendar system from the
+ * era, year-of-era, month-of-year and day-of-month fields.
+ * <p>
+ * Dates in the middle of the cutover gap, such as the 10th September 1752,
+ * will not throw an exception. Instead, the date will be treated as a Julian date
+ * and converted to an ISO date, with the day of month shifted by 11 days.
+ *
+ * @param era the British Cutover era, not null
+ * @param yearOfEra the year-of-era
+ * @param month the month-of-year
+ * @param dayOfMonth the day-of-month
+ * @return the British Cutover local date, not null
+ * @throws DateTimeException if unable to create the date
+ * @throws ClassCastException if the {@code era} is not a {@code JulianEra}
+ */
+ @Override
+ public HybridDate date(Era era, int yearOfEra, int month, int dayOfMonth) {
+ return date(prolepticYear(era, yearOfEra), month, dayOfMonth);
+ }
+
+ /**
+ * Obtains a local date in British Cutover calendar system from the
+ * proleptic-year, month-of-year and day-of-month fields.
+ * <p>
+ * Dates in the middle of the cutover gap, such as the 10th September 1752,
+ * will not throw an exception. Instead, the date will be treated as a Julian date
+ * and converted to an ISO date, with the day of month shifted by 11 days.
+ *
+ * @param prolepticYear the proleptic-year
+ * @param month the month-of-year
+ * @param dayOfMonth the day-of-month
+ * @return the British Cutover local date, not null
+ * @throws DateTimeException if unable to create the date
+ */
+ @Override
+ public HybridDate date(int prolepticYear, int month, int dayOfMonth) {
+ return HybridDate.of(prolepticYear, month, dayOfMonth);
+ }
+
+ /**
+ * Obtains a local date in British Cutover calendar system from the
+ * era, year-of-era and day-of-year fields.
+ * <p>
+ * The day-of-year takes into account the cutover, thus there are only 355 days in 1752.
+ *
+ * @param era the British Cutover era, not null
+ * @param yearOfEra the year-of-era
+ * @param dayOfYear the day-of-year
+ * @return the British Cutover local date, not null
+ * @throws DateTimeException if unable to create the date
+ * @throws ClassCastException if the {@code era} is not a {@code JulianEra}
+ */
+ @Override
+ public HybridDate dateYearDay(Era era, int yearOfEra, int dayOfYear) {
+ return dateYearDay(prolepticYear(era, yearOfEra), dayOfYear);
+ }
+
+ /**
+ * Obtains a local date in British Cutover calendar system from the
+ * proleptic-year and day-of-year fields.
+ * <p>
+ * The day-of-year takes into account the cutover, thus there are only 355 days in 1752.
+ *
+ * @param prolepticYear the proleptic-year
+ * @param dayOfYear the day-of-year
+ * @return the British Cutover local date, not null
+ * @throws DateTimeException if unable to create the date
+ */
+ @Override
+ public HybridDate dateYearDay(int prolepticYear, int dayOfYear) {
+ return HybridDate.ofYearDay(prolepticYear, dayOfYear);
+ }
+
+ /**
+ * Obtains a local date in the British Cutover calendar system from the epoch-day.
+ *
+ * @param epochDay the epoch day
+ * @return the British Cutover local date, not null
+ * @throws DateTimeException if unable to create the date
+ */
+ @Override // override with covariant return type
+ public HybridDate dateEpochDay(long epochDay) {
+ return HybridDate.ofEpochDay(epochDay);
+ }
+
+ //-------------------------------------------------------------------------
+ /**
+ * Obtains the current British Cutover local date from the system clock in the default time-zone.
+ * <p>
+ * This will query the {@link Clock#systemDefaultZone() system clock} in the default
+ * time-zone to obtain the current date.
+ * <p>
+ * Using this method will prevent the ability to use an alternate clock for testing
+ * because the clock is hard-coded.
+ *
+ * @return the current British Cutover local date using the system clock and default time-zone, not null
+ * @throws DateTimeException if unable to create the date
+ */
+ @Override // override with covariant return type
+ public HybridDate dateNow() {
+ return HybridDate.now();
+ }
+
+ /**
+ * Obtains the current British Cutover local date from the system clock in the specified time-zone.
+ * <p>
+ * This will query the {@link Clock#system(ZoneId) system clock} to obtain the current date.
+ * Specifying the time-zone avoids dependence on the default time-zone.
+ * <p>
+ * Using this method will prevent the ability to use an alternate clock for testing
+ * because the clock is hard-coded.
+ *
+ * @param zone the zone ID to use, not null
+ * @return the current British Cutover local date using the system clock, not null
+ * @throws DateTimeException if unable to create the date
+ */
+ @Override // override with covariant return type
+ public HybridDate dateNow(ZoneId zone) {
+ return HybridDate.now(zone);
+ }
+
+ /**
+ * Obtains the current British Cutover local date from the specified clock.
+ * <p>
+ * This will query the specified clock to obtain the current date - today.
+ * Using this method allows the use of an alternate clock for testing.
+ * The alternate clock may be introduced using {@link Clock dependency injection}.
+ *
+ * @param clock the clock to use, not null
+ * @return the current British Cutover local date, not null
+ * @throws DateTimeException if unable to create the date
+ */
+ @Override // override with covariant return type
+ public HybridDate dateNow(Clock clock) {
+ return HybridDate.now(clock);
+ }
+
+ //-------------------------------------------------------------------------
+ /**
+ * Obtains a British Cutover local date from another date-time object.
+ *
+ * @param temporal the date-time object to convert, not null
+ * @return the British Cutover local date, not null
+ * @throws DateTimeException if unable to create the date
+ */
+ @Override
+ public HybridDate date(TemporalAccessor temporal) {
+ return HybridDate.from(temporal);
+ }
+
+ /**
+ * Obtains a British Cutover local date-time from another date-time object.
+ *
+ * @param temporal the date-time object to convert, not null
+ * @return the British Cutover local date-time, not null
+ * @throws DateTimeException if unable to create the date-time
+ */
+ @Override
+ @SuppressWarnings("unchecked")
+ public ChronoLocalDateTime<HybridDate> localDateTime(TemporalAccessor temporal) {
+ return (ChronoLocalDateTime<HybridDate>) super.localDateTime(temporal);
+ }
+
+ /**
+ * Obtains a British Cutover zoned date-time from another date-time object.
+ *
+ * @param temporal the date-time object to convert, not null
+ * @return the British Cutover zoned date-time, not null
+ * @throws DateTimeException if unable to create the date-time
+ */
+ @Override
+ @SuppressWarnings("unchecked")
+ public ChronoZonedDateTime<HybridDate> zonedDateTime(TemporalAccessor temporal) {
+ return (ChronoZonedDateTime<HybridDate>) super.zonedDateTime(temporal);
+ }
+
+ /**
+ * Obtains a British Cutover zoned date-time in this chronology from an {@code Instant}.
+ *
+ * @param instant the instant to create the date-time from, not null
+ * @param zone the time-zone, not null
+ * @return the British Cutover zoned date-time, not null
+ * @throws DateTimeException if the result exceeds the supported range
+ */
+ @Override
+ @SuppressWarnings("unchecked")
+ public ChronoZonedDateTime<HybridDate> zonedDateTime(Instant instant, ZoneId zone) {
+ return (ChronoZonedDateTime<HybridDate>) super.zonedDateTime(instant, zone);
+ }
+
+ //-----------------------------------------------------------------------
+ /**
+ * Checks if the specified year is a leap year.
+ * <p>
+ * The result will return the same as {@link JulianChronology#isLeapYear(long)} for
+ * year 1752 and earlier, and {@link IsoChronology#isLeapYear(long)} otherwise.
+ * This method does not validate the year passed in, and only has a
+ * well-defined result for years in the supported range.
+ *
+ * @param prolepticYear the proleptic-year to check, not validated for range
+ * @return true if the year is a leap year
+ */
+ @Override
+ public boolean isLeapYear(long prolepticYear) {
+ if (prolepticYear <= CUTOVER_YEAR) {
+ return JulianChronology.INSTANCE.isLeapYear(prolepticYear);
+ }
+ return IsoChronology.INSTANCE.isLeapYear(prolepticYear);
+ }
+
+ @Override
+ public int prolepticYear(Era era, int yearOfEra) {
+ if (era instanceof JulianEra == false) {
+ throw new ClassCastException("Era must be JulianEra");
+ }
+ return (era == JulianEra.AD ? yearOfEra : 1 - yearOfEra);
+ }
+
+ @Override
+ public JulianEra eraOf(int eraValue) {
+ return JulianEra.of(eraValue);
+ }
+
+ @Override
+ public List<Era> eras() {
+ return Arrays.<Era>asList(JulianEra.values());
+ }
+
+ //-----------------------------------------------------------------------
+ @Override
+ public ValueRange range(ChronoField field) {
+ switch (field) {
+ case DAY_OF_YEAR:
+ return DOY_RANGE;
+ case ALIGNED_WEEK_OF_MONTH:
+ return ALIGNED_WOM_RANGE;
+ case ALIGNED_WEEK_OF_YEAR:
+ return ALIGNED_WOY_RANGE;
+ case PROLEPTIC_MONTH:
+ return PROLEPTIC_MONTH_RANGE;
+ case YEAR_OF_ERA:
+ return YOE_RANGE;
+ case YEAR:
+ return YEAR_RANGE;
+ default:
+ break;
+ }
+ return field.range();
+ }
+
+ //-----------------------------------------------------------------------
+ @Override // override for return type
+ public HybridDate resolveDate(Map<TemporalField, Long> fieldValues, ResolverStyle resolverStyle) {
+ return (HybridDate) super.resolveDate(fieldValues, resolverStyle);
+ }
+
+}
diff --git a/java/core/src/java/org/threeten/extra/chrono/HybridDate.java b/java/core/src/java/org/threeten/extra/chrono/HybridDate.java
new file mode 100644
index 0000000..c2f87c5
--- /dev/null
+++ b/java/core/src/java/org/threeten/extra/chrono/HybridDate.java
@@ -0,0 +1,537 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.threeten.extra.chrono;
+
+import static org.threeten.extra.chrono.HybridChronology.CUTOVER;
+import static org.threeten.extra.chrono.HybridChronology.CUTOVER_DAYS;
+import static org.threeten.extra.chrono.HybridChronology.CUTOVER_YEAR;
+
+import java.io.Serializable;
+import java.time.Clock;
+import java.time.DateTimeException;
+import java.time.LocalDate;
+import java.time.LocalTime;
+import java.time.ZoneId;
+import java.time.chrono.ChronoLocalDate;
+import java.time.chrono.ChronoLocalDateTime;
+import java.time.chrono.ChronoPeriod;
+import java.time.temporal.ChronoField;
+import java.time.temporal.Temporal;
+import java.time.temporal.TemporalAccessor;
+import java.time.temporal.TemporalAdjuster;
+import java.time.temporal.TemporalAmount;
+import java.time.temporal.TemporalField;
+import java.time.temporal.TemporalQueries;
+import java.time.temporal.TemporalQuery;
+import java.time.temporal.TemporalUnit;
+import java.time.temporal.ValueRange;
+import java.util.Objects;
+
+/**
+ * A date in the British Cutover calendar system.
+ * <p>
+ * This date operates using the {@linkplain HybridChronology British Cutover calendar}.
+ *
+ * <h3>Implementation Requirements</h3>
+ * This class is immutable and thread-safe.
+ * <p>
+ * This class must be treated as a value type. Do not synchronize, rely on the
+ * identity hash code or use the distinction between equals() and ==.
+ */
+public final class HybridDate
+ extends AbstractDate
+ implements ChronoLocalDate, Serializable {
+ /**
+ * Serialization version.
+ */
+ private static final long serialVersionUID = -9626278512674L;
+ /**
+ * The underlying date.
+ */
+ private final LocalDate isoDate;
+ /**
+ * The underlying Julian date if before the cutover.
+ */
+ private final transient JulianDate julianDate;
+
+ //-----------------------------------------------------------------------
+ /**
+ * Obtains the current {@code HybridDate} from the system clock in the default time-zone.
+ * <p>
+ * This will query the {@link Clock#systemDefaultZone() system clock} in the default
+ * time-zone to obtain the current date.
+ * <p>
+ * Using this method will prevent the ability to use an alternate clock for testing
+ * because the clock is hard-coded.
+ *
+ * @return the current date using the system clock and default time-zone, not null
+ */
+ public static HybridDate now() {
+ return now(Clock.systemDefaultZone());
+ }
+
+ /**
+ * Obtains the current {@code HybridDate} from the system clock in the specified time-zone.
+ * <p>
+ * This will query the {@link Clock#system(ZoneId) system clock} to obtain the current date.
+ * Specifying the time-zone avoids dependence on the default time-zone.
+ * <p>
+ * Using this method will prevent the ability to use an alternate clock for testing
+ * because the clock is hard-coded.
+ *
+ * @param zone the zone ID to use, not null
+ * @return the current date using the system clock, not null
+ */
+ public static HybridDate now(ZoneId zone) {
+ return now(Clock.system(zone));
+ }
+
+ /**
+ * Obtains the current {@code HybridDate} from the specified clock.
+ * <p>
+ * This will query the specified clock to obtain the current date - today.
+ * Using this method allows the use of an alternate clock for testing.
+ * The alternate clock may be introduced using {@linkplain Clock dependency injection}.
+ *
+ * @param clock the clock to use, not null
+ * @return the current date, not null
+ * @throws DateTimeException if the current date cannot be obtained
+ */
+ public static HybridDate now(Clock clock) {
+ return new HybridDate(LocalDate.now(clock));
+ }
+
+ /**
+ * Obtains a {@code HybridDate} representing a date in the British Cutover calendar
+ * system from the proleptic-year, month-of-year and day-of-month fields.
+ * <p>
+ * This returns a {@code HybridDate} with the specified fields.
+ * <p>
+ * Dates in the middle of the cutover gap, such as the 10th September 1752,
+ * will not throw an exception. Instead, the date will be treated as a Julian date
+ * and converted to an ISO date, with the day of month shifted by 11 days.
+ * <p>
+ * Invalid dates, such as September 31st will throw an exception.
+ *
+ * @param prolepticYear the British Cutover proleptic-year
+ * @param month the British Cutover month-of-year, from 1 to 12
+ * @param dayOfMonth the British Cutover day-of-month, from 1 to 31
+ * @return the date in British Cutover calendar system, not null
+ * @throws DateTimeException if the value of any field is out of range,
+ * or if the day-of-month is invalid for the month-year
+ */
+ public static HybridDate of(int prolepticYear, int month, int dayOfMonth) {
+ return HybridDate.create(prolepticYear, month, dayOfMonth);
+ }
+
+ /**
+ * Obtains a {@code HybridDate} from a temporal object.
+ * <p>
+ * This obtains a date in the British Cutover calendar system based on the specified temporal.
+ * A {@code TemporalAccessor} represents an arbitrary set of date and time information,
+ * which this factory converts to an instance of {@code HybridDate}.
+ * <p>
+ * The conversion uses the {@link ChronoField#EPOCH_DAY EPOCH_DAY}
+ * field, which is standardized across calendar systems.
+ * <p>
+ * This method matches the signature of the functional interface {@link TemporalQuery}
+ * allowing it to be used as a query via method reference, {@code HybridDate::from}.
+ *
+ * @param temporal the temporal object to convert, not null
+ * @return the date in British Cutover calendar system, not null
+ * @throws DateTimeException if unable to convert to a {@code HybridDate}
+ */
+ public static HybridDate from(TemporalAccessor temporal) {
+ if (temporal instanceof HybridDate) {
+ return (HybridDate) temporal;
+ }
+ return new HybridDate(LocalDate.from(temporal));
+ }
+
+ //-----------------------------------------------------------------------
+ /**
+ * Obtains a {@code HybridDate} representing a date in the British Cutover calendar
+ * system from the proleptic-year and day-of-year fields.
+ * <p>
+ * This returns a {@code HybridDate} with the specified fields.
+ * The day must be valid for the year, otherwise an exception will be thrown.
+ *
+ * @param prolepticYear the British Cutover proleptic-year
+ * @param dayOfYear the British Cutover day-of-year, from 1 to 366
+ * @return the date in British Cutover calendar system, not null
+ * @throws DateTimeException if the value of any field is out of range,
+ * or if the day-of-year is invalid for the year
+ */
+ static HybridDate ofYearDay(int prolepticYear, int dayOfYear) {
+ if (prolepticYear < CUTOVER_YEAR || (prolepticYear == CUTOVER_YEAR && dayOfYear <= 246)) {
+ JulianDate julian = JulianDate.ofYearDay(prolepticYear, dayOfYear);
+ return new HybridDate(julian);
+ } else if (prolepticYear == CUTOVER_YEAR) {
+ LocalDate iso = LocalDate.ofYearDay(prolepticYear, dayOfYear + CUTOVER_DAYS);
+ return new HybridDate(iso);
+ } else {
+ LocalDate iso = LocalDate.ofYearDay(prolepticYear, dayOfYear);
+ return new HybridDate(iso);
+ }
+ }
+
+ /**
+ * Obtains a {@code HybridDate} representing a date in the British Cutover calendar
+ * system from the epoch-day.
+ *
+ * @param epochDay the epoch day to convert based on 1970-01-01 (ISO)
+ * @return the date in British Cutover calendar system, not null
+ * @throws DateTimeException if the epoch-day is out of range
+ */
+ static HybridDate ofEpochDay(final long epochDay) {
+ return new HybridDate(LocalDate.ofEpochDay(epochDay));
+ }
+
+ /**
+ * Creates a {@code HybridDate} validating the input.
+ *
+ * @param prolepticYear the British Cutover proleptic-year
+ * @param month the British Cutover month-of-year, from 1 to 12
+ * @param dayOfMonth the British Cutover day-of-month, from 1 to 31
+ * @return the date in British Cutover calendar system, not null
+ * @throws DateTimeException if the value of any field is out of range,
+ * or if the day-of-month is invalid for the month-year
+ */
+ static HybridDate create(int prolepticYear, int month, int dayOfMonth) {
+ if (prolepticYear < CUTOVER_YEAR) {
+ JulianDate julian = JulianDate.of(prolepticYear, month, dayOfMonth);
+ return new HybridDate(julian);
+ } else {
+ LocalDate iso = LocalDate.of(prolepticYear, month, dayOfMonth);
+ if (iso.isBefore(CUTOVER)) {
+ JulianDate julian = JulianDate.of(prolepticYear, month, dayOfMonth);
+ return new HybridDate(julian);
+ }
+ return new HybridDate(iso);
+ }
+ }
+
+ //-----------------------------------------------------------------------
+ /**
+ * Creates an instance from an ISO date.
+ *
+ * @param isoDate the standard local date, not null
+ */
+ HybridDate(LocalDate isoDate) {
+ Objects.requireNonNull(isoDate, "isoDate");
+ this.isoDate = isoDate;
+ this.julianDate = (isoDate.isBefore(CUTOVER) ? JulianDate.from(isoDate) : null);
+ }
+
+ /**
+ * Creates an instance from a Julian date.
+ *
+ * @param julianDate the Julian date before the cutover, not null
+ */
+ HybridDate(JulianDate julianDate) {
+ Objects.requireNonNull(julianDate, "julianDate");
+ this.isoDate = LocalDate.from(julianDate);
+ this.julianDate = (isoDate.isBefore(CUTOVER) ? julianDate : null);
+ }
+
+ /**
+ * Validates the object.
+ *
+ * @return the resolved date, not null
+ */
+ private Object readResolve() {
+ return new HybridDate(isoDate);
+ }
+
+ //-----------------------------------------------------------------------
+ private boolean isCutoverYear() {
+ return isoDate.getYear() == CUTOVER_YEAR && isoDate.getDayOfYear() > CUTOVER_DAYS;
+ }
+
+ private boolean isCutoverMonth() {
+ return isoDate.getYear() == CUTOVER_YEAR && isoDate.getMonthValue() == 9 && isoDate.getDayOfMonth() > CUTOVER_DAYS;
+ }
+
+ //-------------------------------------------------------------------------
+ @Override
+ int getAlignedDayOfWeekInMonth() {
+ if (isCutoverMonth() && julianDate == null) {
+ return ((getDayOfMonth() - 1 - CUTOVER_DAYS) % lengthOfWeek()) + 1;
+ }
+ return super.getAlignedDayOfWeekInMonth();
+ }
+
+ @Override
+ int getAlignedWeekOfMonth() {
+ if (isCutoverMonth() && julianDate == null) {
+ return ((getDayOfMonth() - 1 - CUTOVER_DAYS) / lengthOfWeek()) + 1;
+ }
+ return super.getAlignedWeekOfMonth();
+ }
+
+ @Override
+ int getProlepticYear() {
+ return (julianDate != null ? julianDate.getProlepticYear() : isoDate.getYear());
+ }
+
+ @Override
+ int getMonth() {
+ return (julianDate != null ? julianDate.getMonth() : isoDate.getMonthValue());
+ }
+
+ @Override
+ int getDayOfMonth() {
+ return (julianDate != null ? julianDate.getDayOfMonth() : isoDate.getDayOfMonth());
+ }
+
+ @Override
+ int getDayOfYear() {
+ if (julianDate != null) {
+ return julianDate.getDayOfYear();
+ }
+ if (isoDate.getYear() == CUTOVER_YEAR) {
+ return isoDate.getDayOfYear() - CUTOVER_DAYS;
+ }
+ return isoDate.getDayOfYear();
+ }
+
+ @Override
+ public ValueRange rangeChrono(ChronoField field) {
+ switch (field) {
+ case DAY_OF_MONTH:
+ // short length, but value range still 1 to 30
+ if (isCutoverMonth()) {
+ return ValueRange.of(1, 30);
+ }
+ return ValueRange.of(1, lengthOfMonth());
+ case DAY_OF_YEAR:
+ // 1 to 355 in cutover year, otherwise 1 to 365/366
+ return ValueRange.of(1, lengthOfYear());
+ case ALIGNED_WEEK_OF_MONTH:
+ // 1 to 3 in cutover month, otherwise 1 to 4/5
+ return rangeAlignedWeekOfMonth();
+ case ALIGNED_WEEK_OF_YEAR:
+ // 1 to 51 in cutover year, otherwise 1 to 53
+ if (isCutoverYear()) {
+ return ValueRange.of(1, 51);
+ }
+ return ChronoField.ALIGNED_WEEK_OF_YEAR.range();
+ default:
+ return getChronology().range(field);
+ }
+ }
+
+ @Override
+ ValueRange rangeAlignedWeekOfMonth() {
+ if (isCutoverMonth()) {
+ return ValueRange.of(1, 3);
+ }
+ return ValueRange.of(1, getMonth() == 2 && isLeapYear() == false ? 4 : 5);
+ }
+
+ @Override
+ HybridDate resolvePrevious(int year, int month, int dayOfMonth) {
+ switch (month) {
+ case 2:
+ dayOfMonth = Math.min(dayOfMonth, getChronology().isLeapYear(year) ? 29 : 28);
+ break;
+ case 4:
+ case 6:
+ case 9:
+ case 11:
+ dayOfMonth = Math.min(dayOfMonth, 30);
+ break;
+ default:
+ break;
+ }
+ return create(year, month, dayOfMonth);
+ }
+
+ //-----------------------------------------------------------------------
+ /**
+ * Gets the chronology of this date, which is the British Cutover calendar system.
+ * <p>
+ * The {@code Chronology} represents the calendar system in use.
+ * The era and other fields in {@link ChronoField} are defined by the chronology.
+ *
+ * @return the British Cutover chronology, not null
+ */
+ @Override
+ public HybridChronology getChronology() {
+ return HybridChronology.INSTANCE;
+ }
+
+ /**
+ * Gets the era applicable at this date.
+ * <p>
+ * The British Cutover calendar system has two eras, 'AD' and 'BC',
+ * defined by {@link JulianEra}.
+ *
+ * @return the era applicable at this date, not null
+ */
+ @Override
+ public JulianEra getEra() {
+ return (getProlepticYear() >= 1 ? JulianEra.AD : JulianEra.BC);
+ }
+
+ /**
+ * Returns the length of the month represented by this date.
+ * <p>
+ * This returns the length of the month in days.
+ * This takes into account the cutover, returning 19 in September 1752.
+ *
+ * @return the length of the month in days, from 19 to 31
+ */
+ @Override
+ public int lengthOfMonth() {
+ if (isCutoverMonth()) {
+ return 19;
+ }
+ return (julianDate != null ? julianDate.lengthOfMonth() : isoDate.lengthOfMonth());
+ }
+
+ /**
+ * Returns the length of the year represented by this date.
+ * <p>
+ * This returns the length of the year in days.
+ * This takes into account the cutover, returning 355 in 1752.
+ *
+ * @return the length of the month in days, from 19 to 31
+ */
+ @Override
+ public int lengthOfYear() {
+ if (isCutoverYear()) {
+ return 355;
+ }
+ return (julianDate != null ? julianDate.lengthOfYear() : isoDate.lengthOfYear());
+ }
+
+ //-------------------------------------------------------------------------
+ @Override
+ public HybridDate with(TemporalAdjuster adjuster) {
+ return (HybridDate) adjuster.adjustInto(this);
+ }
+
+ @Override
+ public HybridDate with(TemporalField field, long newValue) {
+ return (HybridDate) super.with(field, newValue);
+ }
+
+ //-----------------------------------------------------------------------
+ @Override
+ public HybridDate plus(TemporalAmount amount) {
+ return (HybridDate) amount.addTo(this);
+ }
+
+ @Override
+ public HybridDate plus(long amountToAdd, TemporalUnit unit) {
+ return (HybridDate) super.plus(amountToAdd, unit);
+ }
+
+ @Override
+ public HybridDate minus(TemporalAmount amount) {
+ return (HybridDate) amount.subtractFrom(this);
+ }
+
+ @Override
+ public HybridDate minus(long amountToSubtract, TemporalUnit unit) {
+ return (amountToSubtract == Long.MIN_VALUE ? plus(Long.MAX_VALUE, unit).plus(1, unit) : plus(-amountToSubtract, unit));
+ }
+
+ //-------------------------------------------------------------------------
+ @Override // for covariant return type
+ @SuppressWarnings("unchecked")
+ public ChronoLocalDateTime<HybridDate> atTime(LocalTime localTime) {
+ return (ChronoLocalDateTime<HybridDate>) super.atTime(localTime);
+ }
+
+ @Override
+ public long until(Temporal endExclusive, TemporalUnit unit) {
+ return super.until(HybridDate.from(endExclusive), unit);
+ }
+
+ @Override
+ public ChronoPeriod until(ChronoLocalDate endDateExclusive) {
+ HybridDate end = HybridDate.from(endDateExclusive);
+ long totalMonths = end.getProlepticMonth() - this.getProlepticMonth(); // safe
+ int days = end.getDayOfMonth() - this.getDayOfMonth();
+ if (totalMonths == 0 && isCutoverMonth()) {
+ if (julianDate != null && end.julianDate == null) {
+ days -= CUTOVER_DAYS;
+ } else if (julianDate == null && end.julianDate != null) {
+ days += CUTOVER_DAYS;
+ }
+ } else if (totalMonths > 0) {
+ if (julianDate != null && end.julianDate == null) {
+ AbstractDate calcDate = this.plusMonths(totalMonths);
+ days = (int) (end.toEpochDay() - calcDate.toEpochDay()); // safe
+ }
+ if (days < 0) {
+ totalMonths--;
+ AbstractDate calcDate = this.plusMonths(totalMonths);
+ days = (int) (end.toEpochDay() - calcDate.toEpochDay()); // safe
+ }
+ } else if (totalMonths < 0 && days > 0) {
+ totalMonths++;
+ AbstractDate calcDate = this.plusMonths(totalMonths);
+ days = (int) (end.toEpochDay() - calcDate.toEpochDay()); // safe
+ }
+ int years = Math.toIntExact(totalMonths / lengthOfYearInMonths()); // safe
+ int months = (int) (totalMonths % lengthOfYearInMonths()); // safe
+ return getChronology().period(years, months, days);
+ }
+
+ //-----------------------------------------------------------------------
+ @Override
+ public long toEpochDay() {
+ return isoDate.toEpochDay();
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public <R> R query(TemporalQuery<R> query) {
+ if (query == TemporalQueries.localDate()) {
+ return (R) isoDate;
+ }
+ return super.query(query);
+ }
+
+ //-------------------------------------------------------------------------
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (obj instanceof HybridDate) {
+ HybridDate otherDate = (HybridDate) obj;
+ return this.isoDate.equals(otherDate.isoDate);
+ }
+ return false;
+ }
+
+ /**
+ * A hash code for this date.
+ *
+ * @return a suitable hash code based only on the Chronology and the date
+ */
+ @Override
+ public int hashCode() {
+ return getChronology().getId().hashCode() ^ isoDate.hashCode();
+ }
+
+}
diff --git a/java/core/src/test/org/apache/orc/TestProlepticConversions.java b/java/core/src/test/org/apache/orc/TestProlepticConversions.java
index fa1719b..95bcb71 100644
--- a/java/core/src/test/org/apache/orc/TestProlepticConversions.java
+++ b/java/core/src/test/org/apache/orc/TestProlepticConversions.java
@@ -20,9 +20,14 @@
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector;
import org.apache.hadoop.hive.ql.exec.vector.DateColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.Decimal64ColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.DoubleColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector;
import org.apache.hadoop.hive.ql.exec.vector.TimestampColumnVector;
import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
+import org.apache.orc.impl.DateUtils;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
@@ -31,6 +36,7 @@
import org.junit.runners.Parameterized;
import java.io.File;
+import java.nio.charset.StandardCharsets;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Collection;
@@ -43,6 +49,10 @@
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
+/**
+ * This class tests all of the combinations of reading and writing the hybrid
+ * and proleptic calendars.
+ */
@RunWith(Parameterized.class)
public class TestProlepticConversions {
@@ -187,4 +197,181 @@
}
}
}
+
+ /**
+ * Test all of the type conversions from/to date.
+ */
+ @Test
+ public void testSchemaEvolutionDate() throws Exception {
+ TypeDescription schema = TypeDescription.fromString(
+ "struct<d2s:date,d2t:date,s2d:string,t2d:timestamp>");
+ try (Writer writer = OrcFile.createWriter(testFilePath,
+ OrcFile.writerOptions(conf)
+ .setSchema(schema)
+ .fileSystem(fs)
+ .setProlepticGregorian(writerProlepticGregorian)
+ .useUTCTimestamp(true))) {
+ VectorizedRowBatch batch = schema.createRowBatchV2();
+ batch.size = 1024;
+ DateColumnVector d2s = (DateColumnVector) batch.cols[0];
+ DateColumnVector d2t = (DateColumnVector) batch.cols[1];
+ BytesColumnVector s2d = (BytesColumnVector) batch.cols[2];
+ TimestampColumnVector t2d = (TimestampColumnVector) batch.cols[3];
+ d2s.changeCalendar(writerProlepticGregorian, false);
+ d2t.changeCalendar(writerProlepticGregorian, false);
+ t2d.changeCalendar(writerProlepticGregorian, false);
+ GregorianCalendar cal = writerProlepticGregorian ? PROLEPTIC : HYBRID;
+ SimpleDateFormat dateFormat = createParser("yyyy-MM-dd", cal);
+ SimpleDateFormat timeFormat = createParser("yyyy-MM-dd HH:mm:ss", cal);
+ for(int r=0; r < batch.size; ++r) {
+ String date = String.format("%04d-01-23", r * 2 + 1);
+ String time = String.format("%04d-03-21 %02d:12:34", 2 * r + 1, r % 24);
+ d2s.vector[r] = TimeUnit.MILLISECONDS.toDays(dateFormat.parse(date).getTime());
+ d2t.vector[r] = d2s.vector[r];
+ s2d.setVal(r, date.getBytes(StandardCharsets.UTF_8));
+ t2d.time[r] = timeFormat.parse(time).getTime();
+ t2d.nanos[r] = 0;
+ }
+ writer.addRowBatch(batch);
+ }
+ TypeDescription readerSchema = TypeDescription.fromString(
+ "struct<d2s:string,d2t:timestamp,s2d:date,t2d:date>");
+ try (Reader reader = OrcFile.createReader(testFilePath,
+ OrcFile.readerOptions(conf)
+ .filesystem(fs)
+ .convertToProlepticGregorian(readerProlepticGregorian)
+ .useUTCTimestamp(true));
+ RecordReader rows = reader.rows(reader.options()
+ .schema(readerSchema))) {
+ assertEquals(writerProlepticGregorian, reader.writerUsedProlepticGregorian());
+ VectorizedRowBatch batch = readerSchema.createRowBatchV2();
+ BytesColumnVector d2s = (BytesColumnVector) batch.cols[0];
+ TimestampColumnVector d2t = (TimestampColumnVector) batch.cols[1];
+ DateColumnVector s2d = (DateColumnVector) batch.cols[2];
+ DateColumnVector t2d = (DateColumnVector) batch.cols[3];
+ GregorianCalendar cal = readerProlepticGregorian ? PROLEPTIC : HYBRID;
+ SimpleDateFormat dateFormat = createParser("yyyy-MM-dd", cal);
+ SimpleDateFormat timeFormat = createParser("yyyy-MM-dd HH:mm:ss", cal);
+
+ // Check the data
+ assertTrue(rows.nextBatch(batch));
+ assertEquals(1024, batch.size);
+ // Ensure the column vectors are using the right calendar
+ assertEquals(readerProlepticGregorian, d2t.usingProlepticCalendar());
+ assertEquals(readerProlepticGregorian, s2d.isUsingProlepticCalendar());
+ assertEquals(readerProlepticGregorian, t2d.isUsingProlepticCalendar());
+ for(int r=0; r < batch.size; ++r) {
+ String expectedD1 = String.format("%04d-01-23", 2 * r + 1);
+ String expectedD2 = expectedD1 + " 00:00:00";
+ String expectedT = String.format("%04d-03-21", 2 * r + 1);
+ assertEquals("row " + r, expectedD1, d2s.toString(r));
+ assertEquals("row " + r, expectedD2, timeFormat.format(d2t.asScratchTimestamp(r)));
+ assertEquals("row " + r, expectedD1, DateUtils.printDate((int) s2d.vector[r],
+ readerProlepticGregorian));
+ assertEquals("row " + r, expectedT, dateFormat.format(
+ new Date(TimeUnit.DAYS.toMillis(t2d.vector[r]))));
+ }
+ assertEquals(false, rows.nextBatch(batch));
+ }
+ }
+
+ /**
+ * Test all of the type conversions from/to timestamp, except for date,
+ * which was handled above.
+ */
+ @Test
+ public void testSchemaEvolutionTimestamp() throws Exception {
+ TypeDescription schema = TypeDescription.fromString(
+ "struct<t2i:timestamp,t2d:timestamp,t2D:timestamp,t2s:timestamp,"
+ + "i2t:bigint,d2t:decimal(18,2),D2t:double,s2t:string>");
+ try (Writer writer = OrcFile.createWriter(testFilePath,
+ OrcFile.writerOptions(conf)
+ .setSchema(schema)
+ .fileSystem(fs)
+ .setProlepticGregorian(writerProlepticGregorian)
+ .useUTCTimestamp(true))) {
+ VectorizedRowBatch batch = schema.createRowBatchV2();
+ batch.size = 1024;
+ TimestampColumnVector t2i = (TimestampColumnVector) batch.cols[0];
+ TimestampColumnVector t2d = (TimestampColumnVector) batch.cols[1];
+ TimestampColumnVector t2D = (TimestampColumnVector) batch.cols[2];
+ TimestampColumnVector t2s = (TimestampColumnVector) batch.cols[3];
+ LongColumnVector i2t = (LongColumnVector) batch.cols[4];
+ Decimal64ColumnVector d2t = (Decimal64ColumnVector) batch.cols[5];
+ DoubleColumnVector D2t = (DoubleColumnVector) batch.cols[6];
+ BytesColumnVector s2t = (BytesColumnVector) batch.cols[7];
+
+ t2i.changeCalendar(writerProlepticGregorian, false);
+ t2d.changeCalendar(writerProlepticGregorian, false);
+ t2D.changeCalendar(writerProlepticGregorian, false);
+ t2s.changeCalendar(writerProlepticGregorian, false);
+
+ for(int r=0; r < batch.size; ++r) {
+ String time = String.format("%04d-03-21 %02d:12:34.12", 2 * r + 1, r % 24);
+ long millis = DateUtils.parseTime(time, writerProlepticGregorian, true);
+ int nanos = (int) Math.floorMod(millis, 1000) * 1_000_000;
+ t2i.time[r] = millis;
+ t2i.nanos[r] = nanos;
+ t2d.time[r] = millis;
+ t2d.nanos[r] = nanos;
+ t2D.time[r] = millis;
+ t2D.nanos[r] = nanos;
+ t2s.time[r] = millis;
+ d2t.vector[r] = millis / 10;
+ t2s.nanos[r] = nanos;
+ i2t.vector[r] = Math.floorDiv(millis, 1000);
+ d2t.vector[r] = Math.floorDiv(millis, 10);
+ D2t.vector[r] = millis / 1000.0;
+ s2t.setVal(r, time.getBytes(StandardCharsets.UTF_8));
+ }
+ writer.addRowBatch(batch);
+ }
+ TypeDescription readerSchema = TypeDescription.fromString(
+ "struct<i2t:timestamp,d2t:timestamp,D2t:timestamp,s2t:timestamp,"
+ + "t2i:bigint,t2d:decimal(18,2),t2D:double,t2s:string>");
+ try (Reader reader = OrcFile.createReader(testFilePath,
+ OrcFile.readerOptions(conf)
+ .filesystem(fs)
+ .convertToProlepticGregorian(readerProlepticGregorian)
+ .useUTCTimestamp(true));
+ RecordReader rows = reader.rows(reader.options()
+ .schema(readerSchema))) {
+ assertEquals(writerProlepticGregorian, reader.writerUsedProlepticGregorian());
+ VectorizedRowBatch batch = readerSchema.createRowBatchV2();
+ TimestampColumnVector i2t = (TimestampColumnVector) batch.cols[0];
+ TimestampColumnVector d2t = (TimestampColumnVector) batch.cols[1];
+ TimestampColumnVector D2t = (TimestampColumnVector) batch.cols[2];
+ TimestampColumnVector s2t = (TimestampColumnVector) batch.cols[3];
+ LongColumnVector t2i = (LongColumnVector) batch.cols[4];
+ Decimal64ColumnVector t2d = (Decimal64ColumnVector) batch.cols[5];
+ DoubleColumnVector t2D = (DoubleColumnVector) batch.cols[6];
+ BytesColumnVector t2s = (BytesColumnVector) batch.cols[7];
+
+ // Check the data
+ assertTrue(rows.nextBatch(batch));
+ assertEquals(1024, batch.size);
+ // Ensure the column vectors are using the right calendar
+ assertEquals(readerProlepticGregorian, i2t.usingProlepticCalendar());
+ assertEquals(readerProlepticGregorian, d2t.usingProlepticCalendar());
+ assertEquals(readerProlepticGregorian, D2t.usingProlepticCalendar());
+ assertEquals(readerProlepticGregorian, s2t.usingProlepticCalendar());
+ for(int r=0; r < batch.size; ++r) {
+ String time = String.format("%04d-03-21 %02d:12:34.12", 2 * r + 1, r % 24);
+ long millis = DateUtils.parseTime(time, readerProlepticGregorian, true);
+ assertEquals("row " + r, time.substring(0, time.length() - 3),
+ DateUtils.printTime(i2t.time[r], readerProlepticGregorian, true));
+ assertEquals("row " + r, time,
+ DateUtils.printTime(d2t.time[r], readerProlepticGregorian, true));
+ assertEquals("row " + r, time,
+ DateUtils.printTime(D2t.time[r], readerProlepticGregorian, true));
+ assertEquals("row " + r, time,
+ DateUtils.printTime(s2t.time[r], readerProlepticGregorian, true));
+ assertEquals("row " + r, Math.floorDiv(millis, 1000), t2i.vector[r]);
+ assertEquals("row " + r, Math.floorDiv(millis, 10), t2d.vector[r]);
+ assertEquals("row " + r, millis/1000.0, t2D.vector[r], 0.1);
+ assertEquals("row " + r, time, t2s.toString(r));
+ }
+ assertEquals(false, rows.nextBatch(batch));
+ }
+ }
}
diff --git a/java/core/src/test/org/apache/orc/impl/TestDateUtils.java b/java/core/src/test/org/apache/orc/impl/TestDateUtils.java
new file mode 100644
index 0000000..5c75a1d
--- /dev/null
+++ b/java/core/src/test/org/apache/orc/impl/TestDateUtils.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.orc.impl;
+
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+
+public class TestDateUtils {
+ /**
+ * Test case for DateColumnVector's changeCalendar
+ * epoch days, hybrid representation, proleptic representation
+ * 16768: hybrid: 2015-11-29 proleptic: 2015-11-29
+ * -141418: hybrid: 1582-10-24 proleptic: 1582-10-24
+ * -141427: hybrid: 1582-10-15 proleptic: 1582-10-15
+ * -141428: hybrid: 1582-10-04 proleptic: 1582-10-14
+ * -141430: hybrid: 1582-10-02 proleptic: 1582-10-12
+ * -141437: hybrid: 1582-09-25 proleptic: 1582-10-05
+ * -141438: hybrid: 1582-09-24 proleptic: 1582-10-04
+ * -499952: hybrid: 0601-03-04 proleptic: 0601-03-07
+ * -499955: hybrid: 0601-03-01 proleptic: 0601-03-04
+ * @throws Exception
+ */
+ @Test
+ public void testConversion() throws Exception {
+ checkConversion(16768, "2015-11-29", "2015-11-29");
+ checkConversion(-141418, "1582-10-24", "1582-10-24");
+ checkConversion(-141427, "1582-10-15", "1582-10-15");
+ checkConversion(-141428, "1582-10-04", "1582-10-14");
+ checkConversion(-141430, "1582-10-02", "1582-10-12");
+ checkConversion(-141437, "1582-09-25", "1582-10-05");
+ checkConversion(-499952, "0601-03-04", "0601-03-07");
+ checkConversion(-499955, "0601-03-01", "0601-03-04");
+ }
+
+ void checkConversion(int dayOfEpoch, String hybrid, String proleptic) {
+ String result = DateUtils.printDate(dayOfEpoch, false);
+ assertEquals("day " + dayOfEpoch, hybrid, result);
+ assertEquals(dayOfEpoch, (int) DateUtils.parseDate(result, false));
+ result = DateUtils.printDate(dayOfEpoch, true);
+ assertEquals("day " + dayOfEpoch, proleptic, result);
+ assertEquals(dayOfEpoch, (int) DateUtils.parseDate(result, true));
+ }
+}
diff --git a/java/core/src/test/org/apache/orc/impl/TestSchemaEvolution.java b/java/core/src/test/org/apache/orc/impl/TestSchemaEvolution.java
index b9e9f13..a0981a6 100644
--- a/java/core/src/test/org/apache/orc/impl/TestSchemaEvolution.java
+++ b/java/core/src/test/org/apache/orc/impl/TestSchemaEvolution.java
@@ -644,18 +644,27 @@
TimestampColumnVector tcv = new TimestampColumnVector(1024);
batch.cols[0] = tcv;
batch.reset();
- batch.size = 1;
+ batch.size = 3;
tcv.time[0] = 74000L;
+ tcv.nanos[0] = 123456789;
+ tcv.time[1] = 123000L;
+ tcv.nanos[1] = 456000000;
+ tcv.time[2] = 987000;
+ tcv.nanos[2] = 0;
writer.addRowBatch(batch);
writer.close();
Reader reader = OrcFile.createReader(testFilePath,
OrcFile.readerOptions(conf).filesystem(fs));
- TypeDescription schemaOnRead = TypeDescription.createDecimal().withPrecision(38).withScale(1);
+ TypeDescription schemaOnRead = TypeDescription.createDecimal().withPrecision(38).withScale(9);
RecordReader rows = reader.rows(reader.options().schema(schemaOnRead));
batch = schemaOnRead.createRowBatch();
- rows.nextBatch(batch);
- assertEquals("74", ((DecimalColumnVector) batch.cols[0]).vector[0].toString());
+ assertEquals(true, rows.nextBatch(batch));
+ assertEquals(3, batch.size);
+ DecimalColumnVector dcv = (DecimalColumnVector) batch.cols[0];
+ assertEquals("74.123456789", dcv.vector[0].toString());
+ assertEquals("123.456", dcv.vector[1].toString());
+ assertEquals("987", dcv.vector[2].toString());
rows.close();
}
@@ -687,6 +696,40 @@
}
@Test
+ public void testTimestampToStringEvolution() throws Exception {
+ testFilePath = new Path(workDir, "TestOrcFile." +
+ testCaseName.getMethodName() + ".orc");
+ TypeDescription schema = TypeDescription.fromString("timestamp");
+ Writer writer = OrcFile.createWriter(testFilePath,
+ OrcFile.writerOptions(conf).setSchema(schema).stripeSize(100000)
+ .bufferSize(10000).useUTCTimestamp(true));
+ VectorizedRowBatch batch = schema.createRowBatchV2();
+ TimestampColumnVector tcv = (TimestampColumnVector) batch.cols[0];
+ batch.size = 3;
+ tcv.time[0] = 74000L;
+ tcv.nanos[0] = 123456789;
+ tcv.time[1] = 123000L;
+ tcv.nanos[1] = 456000000;
+ tcv.time[2] = 987000;
+ tcv.nanos[2] = 0;
+ writer.addRowBatch(batch);
+ writer.close();
+
+ schema = TypeDescription.fromString("string");
+ Reader reader = OrcFile.createReader(testFilePath,
+ OrcFile.readerOptions(conf).filesystem(fs));
+ RecordReader rows = reader.rows(reader.options().schema(schema));
+ batch = schema.createRowBatchV2();
+ BytesColumnVector bcv = (BytesColumnVector) batch.cols[0];
+ assertEquals(true, rows.nextBatch(batch));
+ assertEquals(3, batch.size);
+ assertEquals("1970-01-01 00:01:14.123456789", bcv.toString(0));
+ assertEquals("1970-01-01 00:02:03.456", bcv.toString(1));
+ assertEquals("1970-01-01 00:16:27", bcv.toString(2));
+ rows.close();
+ }
+
+ @Test
public void testSafePpdEvaluation() throws IOException {
TypeDescription fileStruct1 = TypeDescription.createStruct()
.addField("f1", TypeDescription.createInt())
@@ -1224,6 +1267,7 @@
original = fileType.getChildren().get(1);
assertSame(original, mapped);
}
+
@Test
public void testCaseMismatchInReaderAndWriterSchema() {
TypeDescription fileType =
@@ -1679,19 +1723,23 @@
// place.
static String decimalTimestampToString(long centiseconds, ZoneId zone) {
- long sec = centiseconds / 100;
- int nano = (int) ((centiseconds % 100) * 10_000_000);
- return timestampToString(sec, nano, zone);
+ int nano = (int) (Math.floorMod(centiseconds, 100) * 10_000_000);
+ return timestampToString(centiseconds * 10, nano, zone);
}
static String doubleTimestampToString(double seconds, ZoneId zone) {
- long sec = (long) seconds;
+ long sec = (long) Math.floor(seconds);
int nano = 1_000_000 * (int) Math.round((seconds - sec) * 1000);
- return timestampToString(sec, nano, zone);
+ return timestampToString(sec * 1000, nano, zone);
}
- static String timestampToString(long seconds, int nanos, ZoneId zone) {
- return timestampToString(Instant.ofEpochSecond(seconds, nanos), zone);
+ static String timestampToString(long millis, int nanos, ZoneId zone) {
+ return timestampToString(Instant.ofEpochSecond(Math.floorDiv(millis, 1000),
+ nanos), zone);
+ }
+
+ static String longTimestampToString(long seconds, ZoneId zone) {
+ return timestampToString(Instant.ofEpochSecond(seconds), zone);
}
static String timestampToString(Instant time, ZoneId zone) {
@@ -1794,11 +1842,11 @@
current = 0;
}
assertEquals("row " + r, (timeStrings[r] + " " +
- READER_ZONE.getId()).replace(".7 ", ".0 "),
- timestampToString(t1.vector[current], 0, READER_ZONE));
+ READER_ZONE.getId()).replace(".7 ", " "),
+ longTimestampToString(t1.vector[current], READER_ZONE));
assertEquals("row " + r, (timeStrings[r] + " " +
- WRITER_ZONE.getId()).replace(".7 ", ".0 "),
- timestampToString(t2.vector[current], 0, WRITER_ZONE));
+ WRITER_ZONE.getId()).replace(".7 ", " "),
+ longTimestampToString(t2.vector[current], WRITER_ZONE));
current += 1;
}
assertEquals(false, rows.nextBatch(batch));
@@ -1907,10 +1955,10 @@
current = 0;
}
assertEquals("row " + r, timeStrings[r] + " " + READER_ZONE.getId(),
- timestampToString(timeT1.time[current] / 1000, timeT1.nanos[current], READER_ZONE));
+ timestampToString(timeT1.time[current], timeT1.nanos[current], READER_ZONE));
assertEquals("row " + r,
timestampToString(Instant.from(WRITER_FORMAT.parse(timeStrings[r])), READER_ZONE),
- timestampToString(timeT2.time[current] / 1000, timeT2.nanos[current], READER_ZONE));
+ timestampToString(timeT2.time[current], timeT2.nanos[current], READER_ZONE));
current += 1;
}
assertEquals(false, rows.nextBatch(batch));
@@ -1937,11 +1985,11 @@
current = 0;
}
assertEquals("row " + r, (timeStrings[r] + " " +
- UTC.getId()).replace(".7 ", ".0 "),
- timestampToString(t1.vector[current], 0, UTC));
+ UTC.getId()).replace(".7 ", " "),
+ longTimestampToString(t1.vector[current], UTC));
assertEquals("row " + r, (timeStrings[r] + " " +
- WRITER_ZONE.getId()).replace(".7 ", ".0 "),
- timestampToString(t2.vector[current], 0, WRITER_ZONE));
+ WRITER_ZONE.getId()).replace(".7 ", " "),
+ longTimestampToString(t2.vector[current], WRITER_ZONE));
current += 1;
}
assertEquals(false, rows.nextBatch(batch));
@@ -2049,10 +2097,10 @@
current = 0;
}
assertEquals("row " + r, timeStrings[r] + " UTC",
- timestampToString(timeT1.time[current] / 1000, timeT1.nanos[current], UTC));
+ timestampToString(timeT1.time[current], timeT1.nanos[current], UTC));
assertEquals("row " + r,
timestampToString(Instant.from(WRITER_FORMAT.parse(timeStrings[r])), UTC),
- timestampToString(timeT2.time[current] / 1000, timeT2.nanos[current], UTC));
+ timestampToString(timeT2.time[current], timeT2.nanos[current], UTC));
current += 1;
}
assertEquals(false, rows.nextBatch(batch));
@@ -2069,6 +2117,7 @@
String[] values) throws IOException {
TypeDescription fileSchema =
TypeDescription.fromString("struct<l1:bigint,l2:bigint," +
+ "t1:tinyint,t2:tinyint," +
"d1:decimal(14,2),d2:decimal(14,2)," +
"dbl1:double,dbl2:double," +
"dt1:date,dt2:date," +
@@ -2086,20 +2135,24 @@
int batchSize = batch.getMaxSize();
LongColumnVector l1 = (LongColumnVector) batch.cols[0];
LongColumnVector l2 = (LongColumnVector) batch.cols[1];
- Decimal64ColumnVector d1 = (Decimal64ColumnVector) batch.cols[2];
- Decimal64ColumnVector d2 = (Decimal64ColumnVector) batch.cols[3];
- DoubleColumnVector dbl1 = (DoubleColumnVector) batch.cols[4];
- DoubleColumnVector dbl2 = (DoubleColumnVector) batch.cols[5];
- LongColumnVector dt1 = (LongColumnVector) batch.cols[6];
- LongColumnVector dt2 = (LongColumnVector) batch.cols[7];
- BytesColumnVector s1 = (BytesColumnVector) batch.cols[8];
- BytesColumnVector s2 = (BytesColumnVector) batch.cols[9];
+ LongColumnVector t1 = (LongColumnVector) batch.cols[2];
+ LongColumnVector t2 = (LongColumnVector) batch.cols[3];
+ Decimal64ColumnVector d1 = (Decimal64ColumnVector) batch.cols[4];
+ Decimal64ColumnVector d2 = (Decimal64ColumnVector) batch.cols[5];
+ DoubleColumnVector dbl1 = (DoubleColumnVector) batch.cols[6];
+ DoubleColumnVector dbl2 = (DoubleColumnVector) batch.cols[7];
+ LongColumnVector dt1 = (LongColumnVector) batch.cols[8];
+ LongColumnVector dt2 = (LongColumnVector) batch.cols[9];
+ BytesColumnVector s1 = (BytesColumnVector) batch.cols[10];
+ BytesColumnVector s2 = (BytesColumnVector) batch.cols[11];
for (int r = 0; r < values.length; ++r) {
int row = batch.size++;
Instant utcTime = Instant.from(UTC_FORMAT.parse(values[r]));
Instant writerTime = Instant.from(WRITER_FORMAT.parse(values[r]));
l1.vector[row] = utcTime.getEpochSecond();
l2.vector[row] = writerTime.getEpochSecond();
+ t1.vector[row] = r % 128;
+ t2.vector[row] = r % 128;
// balance out the 2 digits of scale
d1.vector[row] = utcTime.toEpochMilli() / 10;
d2.vector[row] = writerTime.toEpochMilli() / 10;
@@ -2166,6 +2219,8 @@
TypeDescription readerSchema = TypeDescription.fromString(
"struct<l1:timestamp," +
"l2:timestamp with local time zone," +
+ "t1:timestamp," +
+ "t2:timestamp with local time zone," +
"d1:timestamp," +
"d2:timestamp with local time zone," +
"dbl1:timestamp," +
@@ -2177,17 +2232,19 @@
VectorizedRowBatch batch = readerSchema.createRowBatchV2();
TimestampColumnVector l1 = (TimestampColumnVector) batch.cols[0];
TimestampColumnVector l2 = (TimestampColumnVector) batch.cols[1];
- TimestampColumnVector d1 = (TimestampColumnVector) batch.cols[2];
- TimestampColumnVector d2 = (TimestampColumnVector) batch.cols[3];
- TimestampColumnVector dbl1 = (TimestampColumnVector) batch.cols[4];
- TimestampColumnVector dbl2 = (TimestampColumnVector) batch.cols[5];
- TimestampColumnVector dt1 = (TimestampColumnVector) batch.cols[6];
- TimestampColumnVector dt2 = (TimestampColumnVector) batch.cols[7];
- TimestampColumnVector s1 = (TimestampColumnVector) batch.cols[8];
- TimestampColumnVector s2 = (TimestampColumnVector) batch.cols[9];
+ TimestampColumnVector t1 = (TimestampColumnVector) batch.cols[2];
+ TimestampColumnVector t2 = (TimestampColumnVector) batch.cols[3];
+ TimestampColumnVector d1 = (TimestampColumnVector) batch.cols[4];
+ TimestampColumnVector d2 = (TimestampColumnVector) batch.cols[5];
+ TimestampColumnVector dbl1 = (TimestampColumnVector) batch.cols[6];
+ TimestampColumnVector dbl2 = (TimestampColumnVector) batch.cols[7];
+ TimestampColumnVector dt1 = (TimestampColumnVector) batch.cols[8];
+ TimestampColumnVector dt2 = (TimestampColumnVector) batch.cols[9];
+ TimestampColumnVector s1 = (TimestampColumnVector) batch.cols[10];
+ TimestampColumnVector s2 = (TimestampColumnVector) batch.cols[11];
OrcFile.ReaderOptions options = OrcFile.readerOptions(conf);
Reader.Options rowOptions = new Reader.Options().schema(readerSchema);
-
+ int offset = READER_ZONE.getRules().getOffset(Instant.ofEpochSecond(0, 0)).getTotalSeconds();
try (Reader reader = OrcFile.createReader(testFilePath, options);
RecordReader rows = reader.rows(rowOptions)) {
int current = 0;
@@ -2199,39 +2256,45 @@
String expected1 = timeStrings[r] + " " + READER_ZONE.getId();
String expected2 = timeStrings[r] + " " + WRITER_ZONE.getId();
- String midnight = timeStrings[r].substring(0, 10) + " 00:00:00.0";
+ String midnight = timeStrings[r].substring(0, 10) + " 00:00:00";
String expectedDate1 = midnight + " " + READER_ZONE.getId();
String expectedDate2 = midnight + " " + UTC.getId();
- assertEquals("row " + r, expected1.replace(".1 ", ".0 "),
- timestampToString(l1.time[current] / 1000, l1.nanos[current], READER_ZONE));
+ assertEquals("row " + r, expected1.replace(".1 ", " "),
+ timestampToString(l1.time[current], l1.nanos[current], READER_ZONE));
- assertEquals("row " + r, expected2.replace(".1 ", ".0 "),
- timestampToString(l2.time[current] / 1000, l2.nanos[current], WRITER_ZONE));
+ assertEquals("row " + r, expected2.replace(".1 ", " "),
+ timestampToString(l2.time[current], l2.nanos[current], WRITER_ZONE));
+
+ assertEquals("row " + r, longTimestampToString(((r % 128) - offset), READER_ZONE),
+ timestampToString(t1.time[current], t1.nanos[current], READER_ZONE));
+
+ assertEquals("row " + r, longTimestampToString((r % 128), WRITER_ZONE),
+ timestampToString(t2.time[current], t2.nanos[current], WRITER_ZONE));
assertEquals("row " + r, expected1,
- timestampToString(d1.time[current] / 1000, d1.nanos[current], READER_ZONE));
+ timestampToString(d1.time[current], d1.nanos[current], READER_ZONE));
assertEquals("row " + r, expected2,
- timestampToString(d2.time[current] / 1000, d2.nanos[current], WRITER_ZONE));
+ timestampToString(d2.time[current], d2.nanos[current], WRITER_ZONE));
assertEquals("row " + r, expected1,
- timestampToString(dbl1.time[current] / 1000, dbl1.nanos[current], READER_ZONE));
+ timestampToString(dbl1.time[current], dbl1.nanos[current], READER_ZONE));
assertEquals("row " + r, expected2,
- timestampToString(dbl2.time[current] / 1000, dbl2.nanos[current], WRITER_ZONE));
+ timestampToString(dbl2.time[current], dbl2.nanos[current], WRITER_ZONE));
assertEquals("row " + r, expectedDate1,
- timestampToString(dt1.time[current] / 1000, dt1.nanos[current], READER_ZONE));
+ timestampToString(dt1.time[current], dt1.nanos[current], READER_ZONE));
assertEquals("row " + r, expectedDate2,
- timestampToString(dt2.time[current] / 1000, dt2.nanos[current], UTC));
+ timestampToString(dt2.time[current], dt2.nanos[current], UTC));
assertEquals("row " + r, expected1,
- timestampToString(s1.time[current] / 1000, s1.nanos[current], READER_ZONE));
+ timestampToString(s1.time[current], s1.nanos[current], READER_ZONE));
assertEquals("row " + r, expected2,
- timestampToString(s2.time[current] / 1000, s2.nanos[current], WRITER_ZONE));
+ timestampToString(s2.time[current], s2.nanos[current], WRITER_ZONE));
current += 1;
}
assertEquals(false, rows.nextBatch(batch));
@@ -2250,38 +2313,38 @@
String expected1 = timeStrings[r] + " " + UTC.getId();
String expected2 = timeStrings[r] + " " + WRITER_ZONE.getId();
- String midnight = timeStrings[r].substring(0, 10) + " 00:00:00.0";
+ String midnight = timeStrings[r].substring(0, 10) + " 00:00:00";
String expectedDate = midnight + " " + UTC.getId();
- assertEquals("row " + r, expected1.replace(".1 ", ".0 "),
- timestampToString(l1.time[current] / 1000, l1.nanos[current], UTC));
+ assertEquals("row " + r, expected1.replace(".1 ", " "),
+ timestampToString(l1.time[current], l1.nanos[current], UTC));
- assertEquals("row " + r, expected2.replace(".1 ", ".0 "),
- timestampToString(l2.time[current] / 1000, l2.nanos[current], WRITER_ZONE));
+ assertEquals("row " + r, expected2.replace(".1 ", " "),
+ timestampToString(l2.time[current], l2.nanos[current], WRITER_ZONE));
assertEquals("row " + r, expected1,
- timestampToString(d1.time[current] / 1000, d1.nanos[current], UTC));
+ timestampToString(d1.time[current], d1.nanos[current], UTC));
assertEquals("row " + r, expected2,
- timestampToString(d2.time[current] / 1000, d2.nanos[current], WRITER_ZONE));
+ timestampToString(d2.time[current], d2.nanos[current], WRITER_ZONE));
assertEquals("row " + r, expected1,
- timestampToString(dbl1.time[current] / 1000, dbl1.nanos[current], UTC));
+ timestampToString(dbl1.time[current], dbl1.nanos[current], UTC));
assertEquals("row " + r, expected2,
- timestampToString(dbl2.time[current] / 1000, dbl2.nanos[current], WRITER_ZONE));
+ timestampToString(dbl2.time[current], dbl2.nanos[current], WRITER_ZONE));
assertEquals("row " + r, expectedDate,
- timestampToString(dt1.time[current] / 1000, dt1.nanos[current], UTC));
+ timestampToString(dt1.time[current], dt1.nanos[current], UTC));
assertEquals("row " + r, expectedDate,
- timestampToString(dt2.time[current] / 1000, dt2.nanos[current], UTC));
+ timestampToString(dt2.time[current], dt2.nanos[current], UTC));
assertEquals("row " + r, expected1,
- timestampToString(s1.time[current] / 1000, s1.nanos[current], UTC));
+ timestampToString(s1.time[current], s1.nanos[current], UTC));
assertEquals("row " + r, expected2,
- timestampToString(s2.time[current] / 1000, s2.nanos[current], WRITER_ZONE));
+ timestampToString(s2.time[current], s2.nanos[current], WRITER_ZONE));
current += 1;
}
assertEquals(false, rows.nextBatch(batch));
@@ -2296,14 +2359,14 @@
floatAndDoubleToTimeStampOverflow("double",
340282347000000000000000000000000000000000.0,
1e16,
- 9223372036854775.0,
+ 9223372036854778.0,
9000000000000000.1,
10000000000.0,
10000000.123,
-1000000.123,
-10000000000.0,
-9000000000000000.1,
- -9223372036854775.0,
+ -9223372036854778.0,
-1e16,
-340282347000000000000000000000000000000000.0);
}
@@ -2313,14 +2376,14 @@
floatAndDoubleToTimeStampOverflow("float",
340282347000000000000000000000000000000000.0,
1e16,
- 9223372036854775.0,
+ 9223372036854778.0,
9000000000000000.1,
10000000000.0,
10000000.123,
-1000000.123,
-10000000000.0,
-9000000000000000.1,
- -9223372036854775.0,
+ -9223372036854778.0,
-1e16,
-340282347000000000000000000000000000000000.0);
}
@@ -2376,7 +2439,8 @@
assertFalse(rowName, t1.noNulls);
assertTrue(rowName, t1.isNull[row]);
} else {
- double actual = t1.time[row] / 1000.0 + t1.nanos[row] / 1_000_000_000.0;
+ double actual = Math.floorDiv(t1.time[row], 1000) +
+ t1.nanos[row] / 1_000_000_000.0;
assertEquals(rowName, expected, actual,
Math.abs(expected * (isFloat ? 0.000001 : 0.0000000000000001)));
assertFalse(rowName, t1.isNull[row]);
@@ -2392,6 +2456,7 @@
}
}
+ @Test
public void testCheckAcidSchema() {
String ccSchema = "struct<operation:int,originalTransaction:bigint,bucket:int," +
"rowId:bigint,currentTransaction:bigint," +
diff --git a/java/pom.xml b/java/pom.xml
index 5b7c47e..04c20c5 100644
--- a/java/pom.xml
+++ b/java/pom.xml
@@ -686,6 +686,11 @@
<artifactId>threetenbp</artifactId>
<version>1.3.5</version>
</dependency>
+ <dependency>
+ <groupId>org.threeten</groupId>
+ <artifactId>threeten-extra</artifactId>
+ <version>1.5.0</version>
+ </dependency>
<!-- test inter-project -->
<dependency>
diff --git a/java/tools/src/test/resources/orc-file-dump-bloomfilter.out b/java/tools/src/test/resources/orc-file-dump-bloomfilter.out
index d46682b..87b665c 100644
--- a/java/tools/src/test/resources/orc-file-dump-bloomfilter.out
+++ b/java/tools/src/test/resources/orc-file-dump-bloomfilter.out
@@ -178,7 +178,7 @@
Entry 0: numHashFunctions: 4 bitCount: 6272 popCount: 138 loadFactor: 0.022 expectedFpp: 2.343647E-7
Stripe level merge: numHashFunctions: 4 bitCount: 6272 popCount: 138 loadFactor: 0.022 expectedFpp: 2.343647E-7
-File length: 272533 bytes
+File length: 272535 bytes
Padding length: 0 bytes
Padding ratio: 0%
________________________________________________________________________________________________________________________
diff --git a/java/tools/src/test/resources/orc-file-dump-bloomfilter2.out b/java/tools/src/test/resources/orc-file-dump-bloomfilter2.out
index 0943a05..f70ce5f 100644
--- a/java/tools/src/test/resources/orc-file-dump-bloomfilter2.out
+++ b/java/tools/src/test/resources/orc-file-dump-bloomfilter2.out
@@ -188,7 +188,7 @@
Entry 0: numHashFunctions: 7 bitCount: 9600 popCount: 4948 loadFactor: 0.5154 expectedFpp: 0.00966294
Stripe level merge: numHashFunctions: 7 bitCount: 9600 popCount: 4948 loadFactor: 0.5154 expectedFpp: 0.00966294
-File length: 332564 bytes
+File length: 332566 bytes
Padding length: 0 bytes
Padding ratio: 0%
________________________________________________________________________________________________________________________
diff --git a/java/tools/src/test/resources/orc-file-dump-dictionary-threshold.out b/java/tools/src/test/resources/orc-file-dump-dictionary-threshold.out
index 7b96b44..1e6e50e 100644
--- a/java/tools/src/test/resources/orc-file-dump-dictionary-threshold.out
+++ b/java/tools/src/test/resources/orc-file-dump-dictionary-threshold.out
@@ -184,7 +184,7 @@
Row group indices for column 3:
Entry 0: count: 1000 hasNull: false min: Darkness,-230-368-488-586-862-930-1686-2044-2636-2652-2872-3108-3162-3192-3404-3442-3508-3542-3550-3712-3980-4146-4204-4336-4390-4418-4424-4490-4512-4650-4768-4924-4950-5210-5524-5630-5678-5710-5758-5952-6238-6252-6300-6366-6668-6712-6926-6942-7100-7194-7802-8030-8452-8608-8640-8862-8868-9134-9234-9412-9602-9608-9642-9678-9740-9780-10426-10510-10514-10706-10814-10870-10942-11028-11244-11326-11462-11496-11656-11830-12022-12178-12418-12832-13304-13448-13590-13618-13908-14188-14246-14340-14364-14394-14762-14850-14964-15048-15494-15674-15726-16006-16056-16180-16304-16332-16452-16598-16730-16810-16994-17210-17268-17786-17962-18214-18444-18446-18724-18912-18952-19164-19348-19400-19546-19776-19896-20084 max: worst-54-290-346-648-908-996-1038-1080-1560-1584-1620-1744-1770-1798-1852-1966-2162-2244-2286-2296-2534-2660-3114-3676-3788-4068-4150-4706-4744-5350-5420-5582-5696-5726-6006-6020-6024-6098-6184-6568-6636-6802-6994-7004-7318-7498-7758-7780-7798-7920-7952-7960-7988-8232-8256-8390-8416-8478-8620-8840-8984-9038-9128-9236-9248-9344-9594-9650-9714-9928-9938-10178-10368-10414-10502-10732-10876-11008-11158-11410-11722-11836-11964-12054-12096-12126-12136-12202-12246-12298-12616-12774-12782-12790-12802-12976-13216-13246-13502-13766-14454-14974-15004-15124-15252-15294-15356-15530-15610-16316-16936-17024-17122-17214-17310-17528-17682-17742-17870-17878-18010-18410-18524-18788-19204-19254-19518-19596-19786-19874-19904-20390-20752-20936 sum: 670762 positions: 0,0,0,0,0
-File length: 2217710 bytes
+File length: 2217712 bytes
Padding length: 0 bytes
Padding ratio: 0%
________________________________________________________________________________________________________________________
diff --git a/java/tools/src/test/resources/orc-file-dump.json b/java/tools/src/test/resources/orc-file-dump.json
index cc7cccf..3545efe 100644
--- a/java/tools/src/test/resources/orc-file-dump.json
+++ b/java/tools/src/test/resources/orc-file-dump.json
@@ -1361,7 +1361,7 @@
}]
}
],
- "fileLength": 272511,
+ "fileLength": 272513,
"paddingLength": 0,
"paddingRatio": 0,
"status": "OK"
diff --git a/java/tools/src/test/resources/orc-file-dump.out b/java/tools/src/test/resources/orc-file-dump.out
index 1fca5c5..6b9e5f9 100644
--- a/java/tools/src/test/resources/orc-file-dump.out
+++ b/java/tools/src/test/resources/orc-file-dump.out
@@ -189,7 +189,7 @@
Row group indices for column 3:
Entry 0: count: 1000 hasNull: false min: Darkness, max: worst sum: 3866 positions: 0,0,0
-File length: 271047 bytes
+File length: 271049 bytes
Padding length: 0 bytes
Padding ratio: 0%
diff --git a/java/tools/src/test/resources/orc-file-has-null.out b/java/tools/src/test/resources/orc-file-has-null.out
index e1a5413..da850d0 100644
--- a/java/tools/src/test/resources/orc-file-has-null.out
+++ b/java/tools/src/test/resources/orc-file-has-null.out
@@ -106,7 +106,7 @@
Entry 3: count: 0 hasNull: true positions: 0,4,115,0,0,0,0
Entry 4: count: 0 hasNull: true positions: 0,6,110,0,0,0,0
-File length: 1842 bytes
+File length: 1844 bytes
Padding length: 0 bytes
Padding ratio: 0%
________________________________________________________________________________________________________________________
