Revert "GH-3574: Statistics.toParquetStatistics always set null_count(#3575)" (#3688)
This reverts commit 63aebcc0cf3684ba6c9dc2b7243c6007d3a1269f.
diff --git a/parquet-hadoop/src/main/java/org/apache/parquet/format/converter/ParquetMetadataConverter.java b/parquet-hadoop/src/main/java/org/apache/parquet/format/converter/ParquetMetadataConverter.java
index 8600b2c..465516e 100644
--- a/parquet-hadoop/src/main/java/org/apache/parquet/format/converter/ParquetMetadataConverter.java
+++ b/parquet-hadoop/src/main/java/org/apache/parquet/format/converter/ParquetMetadataConverter.java
@@ -815,17 +815,15 @@
public static Statistics toParquetStatistics(
org.apache.parquet.column.statistics.Statistics stats, int truncateLength) {
Statistics formatStats = new Statistics();
- if (!stats.isEmpty()) {
- formatStats.setNull_count(stats.getNumNulls());
- if (stats.isNanCountSet()) {
- formatStats.setNan_count(stats.getNanCount());
- }
- }
// Don't write stats larger than the max size rather than truncating. The
// rationale is that some engines may use the minimum value in the page as
// the true minimum for aggregations and there is no way to mark that a
// value has been truncated and is a lower bound and not in the page.
if (!stats.isEmpty() && withinLimit(stats, truncateLength)) {
+ formatStats.setNull_count(stats.getNumNulls());
+ if (stats.isNanCountSet()) {
+ formatStats.setNan_count(stats.getNanCount());
+ }
if (stats.hasNonNullValue()) {
byte[] min;
byte[] max;
diff --git a/parquet-hadoop/src/test/java/org/apache/parquet/format/converter/TestParquetMetadataConverter.java b/parquet-hadoop/src/test/java/org/apache/parquet/format/converter/TestParquetMetadataConverter.java
index c6deee2..4d361d6 100644
--- a/parquet-hadoop/src/test/java/org/apache/parquet/format/converter/TestParquetMetadataConverter.java
+++ b/parquet-hadoop/src/test/java/org/apache/parquet/format/converter/TestParquetMetadataConverter.java
@@ -878,7 +878,7 @@
}
assertThat(formatStats.getNull_count()).as("Num nulls should match").isEqualTo(3004);
- // min/max are not written because the values are too large, but null count is always written
+ // convert to empty stats because the values are too large
stats.setMinMaxFromBytes(max, max);
formatStats = helper.toParquetStatistics(stats);
@@ -891,7 +891,9 @@
assertThat(formatStats.isSetMax_value())
.as("Max_value should not be set")
.isFalse();
- assertThat(formatStats.getNull_count()).as("Num nulls should match").isEqualTo(3004);
+ assertThat(formatStats.isSetNull_count())
+ .as("Num nulls should not be set")
+ .isFalse();
Statistics roundTripStats = ParquetMetadataConverter.fromParquetStatisticsInternal(
Version.FULL_VERSION,
@@ -901,10 +903,7 @@
assertThat(roundTripStats.isEmpty())
.as("Round-trip stats should not be empty (null count is set)")
- .isFalse();
- assertThat(roundTripStats.getNumNulls())
- .as("Round-trip null count should match")
- .isEqualTo(3004);
+ .isTrue();
}
@Test