ORC-578: Addendum fixing up the stripe statistics
Signed-off-by: Owen O'Malley <omalley@apache.org>
diff --git a/java/core/src/java/org/apache/orc/Reader.java b/java/core/src/java/org/apache/orc/Reader.java
index 9be6f90..9838d89 100644
--- a/java/core/src/java/org/apache/orc/Reader.java
+++ b/java/core/src/java/org/apache/orc/Reader.java
@@ -500,4 +500,9 @@
* Was the file written using the proleptic Gregorian calendar.
*/
boolean writerUsedProlepticGregorian();
+
+ /**
+ * Should the returned values use the proleptic Gregorian calendar?
+ */
+ boolean getConvertToProlepticGregorian();
}
diff --git a/java/core/src/java/org/apache/orc/StripeStatistics.java b/java/core/src/java/org/apache/orc/StripeStatistics.java
index d02bc33..142b8cb 100644
--- a/java/core/src/java/org/apache/orc/StripeStatistics.java
+++ b/java/core/src/java/org/apache/orc/StripeStatistics.java
@@ -19,7 +19,6 @@
package org.apache.orc;
import org.apache.orc.impl.ColumnStatisticsImpl;
-import org.apache.orc.impl.ReaderImpl;
import java.util.List;
@@ -29,19 +28,21 @@
public class StripeStatistics {
protected final List<OrcProto.ColumnStatistics> cs;
protected final TypeDescription schema;
- private final ReaderImpl reader;
+ private final boolean writerUsedProlepticGregorian;
+ private final boolean convertToProlepticGregorian;
- public StripeStatistics(List<OrcProto.ColumnStatistics> list,
- ReaderImpl reader) {
- this(null, list, reader);
+ public StripeStatistics(List<OrcProto.ColumnStatistics> list) {
+ this(null, list, false, false);
}
public StripeStatistics(TypeDescription schema,
List<OrcProto.ColumnStatistics> list,
- ReaderImpl reader) {
+ boolean writerUsedProlepticGregorian,
+ boolean convertToProlepticGregorian) {
this.schema = schema;
this.cs = list;
- this.reader = reader;
+ this.writerUsedProlepticGregorian = writerUsedProlepticGregorian;
+ this.convertToProlepticGregorian = convertToProlepticGregorian;
}
private int getBase() {
@@ -58,9 +59,8 @@
int base = getBase();
for (int c = 0; c < result.length; ++c) {
TypeDescription column = schema == null ? null : schema.findSubtype(base + c);
- result[c] = reader == null
- ? ColumnStatisticsImpl.deserialize(column, cs.get(c))
- : ColumnStatisticsImpl.deserialize(column, cs.get(c), reader);
+ result[c] = ColumnStatisticsImpl.deserialize(column, cs.get(c),
+ writerUsedProlepticGregorian, convertToProlepticGregorian);
}
return result;
}
diff --git a/java/core/src/java/org/apache/orc/impl/ColumnStatisticsImpl.java b/java/core/src/java/org/apache/orc/impl/ColumnStatisticsImpl.java
index f671b5f..263c3bb 100644
--- a/java/core/src/java/org/apache/orc/impl/ColumnStatisticsImpl.java
+++ b/java/core/src/java/org/apache/orc/impl/ColumnStatisticsImpl.java
@@ -2027,13 +2027,6 @@
public static ColumnStatisticsImpl deserialize(TypeDescription schema,
OrcProto.ColumnStatistics stats,
- ReaderImpl reader) {
- return deserialize(schema, stats, reader.writerUsedProlepticGregorian(),
- reader.options.getConvertToProlepticGregorian());
- }
-
- public static ColumnStatisticsImpl deserialize(TypeDescription schema,
- OrcProto.ColumnStatistics stats,
boolean writerUsedProlepticGregorian,
boolean convertToProlepticGregorian) {
if (stats.hasBucketStatistics()) {
diff --git a/java/core/src/java/org/apache/orc/impl/ReaderImpl.java b/java/core/src/java/org/apache/orc/impl/ReaderImpl.java
index 3017a33..fff09db 100644
--- a/java/core/src/java/org/apache/orc/impl/ReaderImpl.java
+++ b/java/core/src/java/org/apache/orc/impl/ReaderImpl.java
@@ -375,7 +375,8 @@
TypeDescription root = encryption.getRoot();
for(int i= 0; i < result.length; ++i){
result[i] = ColumnStatisticsImpl.deserialize(root.findSubtype(root.getId() + i),
- decrypted.getColumn(i), this);
+ decrypted.getColumn(i), writerUsedProlepticGregorian(),
+ getConvertToProlepticGregorian());
}
return result;
}
@@ -388,7 +389,8 @@
for(int i=0; i < result.length; ++i) {
TypeDescription subschema = schema == null ? null : schema.findSubtype(i);
result[i] = ColumnStatisticsImpl.deserialize(subschema, fileStats.get(i),
- this);
+ writerUsedProlepticGregorian(),
+ getConvertToProlepticGregorian());
}
return result;
}
@@ -558,14 +560,6 @@
return OrcFile.WriterVersion.FUTURE;
}
- private static OrcProto.Footer extractFooter(ByteBuffer bb, int footerAbsPos,
- int footerSize, InStream.StreamOptions options) throws IOException {
- bb.position(footerAbsPos);
- bb.limit(footerAbsPos + footerSize);
- return OrcProto.Footer.parseFrom(InStream.createCodedInputStream(
- InStream.create("footer", new BufferChunk(bb, 0), 0, footerSize, options)));
- }
-
public static OrcProto.Metadata extractMetadata(ByteBuffer bb, int metadataAbsPos,
int metadataSize, InStream.StreamOptions options) throws IOException {
bb.position(metadataAbsPos);
@@ -724,8 +718,8 @@
} catch (Throwable thr) {
try {
close();
- } catch (IOException ignore) {
- LOG.info("Ignoring secondary exception in close of " + path, ignore);
+ } catch (IOException except) {
+ LOG.info("Ignoring secondary exception in close of " + path, except);
}
throw thr instanceof IOException ? (IOException) thr :
new IOException("Problem reading file footer " + path, thr);
@@ -748,6 +742,11 @@
}
@Override
+ public boolean getConvertToProlepticGregorian() {
+ return options.getConvertToProlepticGregorian();
+ }
+
+ @Override
public Options options() {
return new Options(conf);
}
@@ -905,7 +904,8 @@
List<StripeStatistics> result = new ArrayList<>(list.size());
for (OrcProto.StripeStatistics ss : stripeStatistics) {
result.add(new StripeStatisticsImpl(schema,
- new ArrayList<>(ss.getColStatsList()), this));
+ new ArrayList<>(ss.getColStatsList()), writerUsedProlepticGregorian(),
+ getConvertToProlepticGregorian()));
}
return result;
}
diff --git a/java/core/src/java/org/apache/orc/impl/StripeStatisticsImpl.java b/java/core/src/java/org/apache/orc/impl/StripeStatisticsImpl.java
index 50557fb..c9f6ee4 100644
--- a/java/core/src/java/org/apache/orc/impl/StripeStatisticsImpl.java
+++ b/java/core/src/java/org/apache/orc/impl/StripeStatisticsImpl.java
@@ -28,12 +28,16 @@
public class StripeStatisticsImpl extends StripeStatistics {
public StripeStatisticsImpl(TypeDescription schema,
List<OrcProto.ColumnStatistics> list,
- ReaderImpl reader) {
- super(schema, list, reader);
+ boolean writerUsedProlepticGregorian,
+ boolean convertToProlepticGregorian) {
+ super(schema, list, writerUsedProlepticGregorian, convertToProlepticGregorian);
}
- public StripeStatisticsImpl(TypeDescription schema, ReaderImpl reader) {
- super(schema, createList(schema), reader);
+ public StripeStatisticsImpl(TypeDescription schema,
+ boolean writerUsedProlepticGregorian,
+ boolean convertToProlepticGregorian) {
+ super(schema, createList(schema), writerUsedProlepticGregorian,
+ convertToProlepticGregorian);
}
/**
diff --git a/java/core/src/java/org/apache/orc/impl/WriterImpl.java b/java/core/src/java/org/apache/orc/impl/WriterImpl.java
index c03e1b4..819e43b 100644
--- a/java/core/src/java/org/apache/orc/impl/WriterImpl.java
+++ b/java/core/src/java/org/apache/orc/impl/WriterImpl.java
@@ -777,7 +777,7 @@
appendStripe(stripe, offset, length, stripeInfo,
new StripeStatistics[]{
new StripeStatisticsImpl(schema, stripeStatistics.getColStatsList(),
- null)});
+ false, false)});
}
@Override
diff --git a/java/core/src/java/org/apache/orc/impl/reader/ReaderEncryptionVariant.java b/java/core/src/java/org/apache/orc/impl/reader/ReaderEncryptionVariant.java
index 97fbd04..c9a687f 100644
--- a/java/core/src/java/org/apache/orc/impl/reader/ReaderEncryptionVariant.java
+++ b/java/core/src/java/org/apache/orc/impl/reader/ReaderEncryptionVariant.java
@@ -236,7 +236,8 @@
) throws IOException {
StripeStatisticsImpl[] result = new StripeStatisticsImpl[stripeCount];
for(int s=0; s < result.length; ++s) {
- result[s] = new StripeStatisticsImpl(column, reader);
+ result[s] = new StripeStatisticsImpl(column, reader.writerUsedProlepticGregorian(),
+ reader.getConvertToProlepticGregorian());
}
// create the objects
long offset = stripeStatsOffset;
diff --git a/java/tools/src/java/org/apache/orc/tools/FileDump.java b/java/tools/src/java/org/apache/orc/tools/FileDump.java
index 92711eb..3a06e5b 100644
--- a/java/tools/src/java/org/apache/orc/tools/FileDump.java
+++ b/java/tools/src/java/org/apache/orc/tools/FileDump.java
@@ -724,7 +724,9 @@
buf.append("no stats at ");
} else {
ColumnStatistics cs =
- ColumnStatisticsImpl.deserialize(colSchema, colStats, reader);
+ ColumnStatisticsImpl.deserialize(colSchema, colStats,
+ reader.writerUsedProlepticGregorian(),
+ reader.getConvertToProlepticGregorian());
buf.append(cs.toString());
}
buf.append(" positions: ");
diff --git a/java/tools/src/java/org/apache/orc/tools/JsonFileDump.java b/java/tools/src/java/org/apache/orc/tools/JsonFileDump.java
index 61849f9..a2664f8 100644
--- a/java/tools/src/java/org/apache/orc/tools/JsonFileDump.java
+++ b/java/tools/src/java/org/apache/orc/tools/JsonFileDump.java
@@ -460,7 +460,8 @@
}
OrcProto.ColumnStatistics colStats = entry.getStatistics();
writeColumnStatistics(writer, ColumnStatisticsImpl.deserialize(
- schema.findSubtype(col), colStats, reader));
+ schema.findSubtype(col), colStats, reader.writerUsedProlepticGregorian(),
+ reader.getConvertToProlepticGregorian()));
writer.key("positions").array();
for (int posIx = 0; posIx < entry.getPositionsCount(); ++posIx) {
writer.value(entry.getPositions(posIx));
