commit c246da995edf50de5349525996ea321c15facbf5
author Korn, Uwe <Uwe.Korn@blue-yonder.com> Thu Jul 26 23:12:11 2018 +0200
committer Uwe L. Korn <uwe@apache.org> Thu Jul 26 23:12:11 2018 +0200
tree e5ca9e08ab233c3a45696399489578d44247ba84
parent 5264ad405a1be2939cd67902f43c626cdf6e8d07

PARQUET-1358: index_page_offset should be unset as it is not supported

Author: Korn, Uwe <Uwe.Korn@blue-yonder.com>

Closes #480 from xhochy/PARQUET-1358 and squashes the following commits:

dcf9a94 [Korn, Uwe] PARQUET-1358: index_page_offset should be unset as it is not supported

src/parquet/column_writer.cc

diff --git a/src/parquet/column_writer.cc b/src/parquet/column_writer.cc
index b3ff8c3..7d47d3f 100644
--- a/src/parquet/column_writer.cc
+++ b/src/parquet/column_writer.cc
@@ -180,8 +180,8 @@
   }
 
   void Close(bool has_dictionary, bool fallback) override {
-    // index_page_offset = 0 since they are not supported
-    metadata_->Finish(num_values_, dictionary_page_offset_, 0, data_page_offset_,
+    // index_page_offset = -1 since they are not supported
+    metadata_->Finish(num_values_, dictionary_page_offset_, -1, data_page_offset_,
                       total_compressed_size_, total_uncompressed_size_, has_dictionary,
                       fallback);
 

src/parquet/file-serialize-test.cc

diff --git a/src/parquet/file-serialize-test.cc b/src/parquet/file-serialize-test.cc
index 16a7c4f..31d2bd4 100644
--- a/src/parquet/file-serialize-test.cc
+++ b/src/parquet/file-serialize-test.cc
@@ -98,6 +98,7 @@
       int64_t values_read;
 
       for (int i = 0; i < num_columns_; ++i) {
+        ASSERT_FALSE(rg_reader->metadata()->ColumnChunk(i)->has_index_page());
         std::vector<int16_t> def_levels_out(rows_per_rowgroup_);
         std::vector<int16_t> rep_levels_out(rows_per_rowgroup_);
         auto col_reader =

src/parquet/metadata.cc

diff --git a/src/parquet/metadata.cc b/src/parquet/metadata.cc
index 49999a4..d9c5d29 100644
--- a/src/parquet/metadata.cc
+++ b/src/parquet/metadata.cc
@@ -151,6 +151,10 @@
 
   inline int64_t data_page_offset() const { return column_->meta_data.data_page_offset; }
 
+  inline bool has_index_page() const {
+    return column_->meta_data.__isset.index_page_offset;
+  }
+
   inline int64_t index_page_offset() const {
     return column_->meta_data.index_page_offset;
   }
@@ -218,6 +222,10 @@
   return impl_->data_page_offset();
 }
 
+bool ColumnChunkMetaData::has_index_page() const {
+  return impl_->has_index_page();
+}
+
 int64_t ColumnChunkMetaData::index_page_offset() const {
   return impl_->index_page_offset();
 }
@@ -607,7 +615,9 @@
     }
     column_chunk_->__isset.meta_data = true;
     column_chunk_->meta_data.__set_num_values(num_values);
-    column_chunk_->meta_data.__set_index_page_offset(index_page_offset);
+    if (index_page_offset >= 0) {
+      column_chunk_->meta_data.__set_index_page_offset(index_page_offset);
+    }
     column_chunk_->meta_data.__set_data_page_offset(data_page_offset);
     column_chunk_->meta_data.__set_total_uncompressed_size(uncompressed_size);
     column_chunk_->meta_data.__set_total_compressed_size(compressed_size);

src/parquet/metadata.h

diff --git a/src/parquet/metadata.h b/src/parquet/metadata.h
index 84c1470..a9739ce 100644
--- a/src/parquet/metadata.h
+++ b/src/parquet/metadata.h
@@ -111,6 +111,7 @@
   bool has_dictionary_page() const;
   int64_t dictionary_page_offset() const;
   int64_t data_page_offset() const;
+  bool has_index_page() const;
   int64_t index_page_offset() const;
   int64_t total_compressed_size() const;
   int64_t total_uncompressed_size() const;