commit 2121150a0d5536865f4acdf8ee440b900d236e06
author Wakahisa <nevilledips@gmail.com> Thu Apr 29 18:26:40 2021 +0200
committer GitHub <noreply@github.com> Thu Apr 29 12:26:40 2021 -0400
tree a7e1c6a5e853d948953642d4ad2b417cd764130f
parent ed00e4d4a160cd5182bfafb81fee2240ec005014

[Parquet] Read list field correctly (#234)

parquet/src/arrow/array_reader.rs

diff --git a/parquet/src/arrow/array_reader.rs b/parquet/src/arrow/array_reader.rs
index a906147..943820f 100644
--- a/parquet/src/arrow/array_reader.rs
+++ b/parquet/src/arrow/array_reader.rs
@@ -1303,6 +1303,15 @@
                 item_type
             ))),
             _ => {
+                // a list is a group type with a single child. The list child's
+                // name comes from the child's field name.
+                let mut list_child = list_type.get_fields().first().ok_or(ArrowError(
+                    "List GroupType should have a field".to_string(),
+                ))?;
+                // if the child's name is "list" and it has a child, then use this child
+                if list_child.name() == "list" && !list_child.get_fields().is_empty() {
+                    list_child = list_child.get_fields().first().unwrap();
+                }
                 let arrow_type = self
                     .arrow_schema
                     .field_with_name(list_type.name())
@@ -1310,9 +1319,9 @@
                     .map(|f| f.data_type().to_owned())
                     .unwrap_or_else(|| {
                         ArrowType::List(Box::new(Field::new(
-                            list_type.name(),
+                            list_child.name(),
                             item_reader_type.clone(),
-                            list_type.is_optional(),
+                            list_child.is_optional(),
                         )))
                     });
 

parquet/src/arrow/arrow_writer.rs

diff --git a/parquet/src/arrow/arrow_writer.rs b/parquet/src/arrow/arrow_writer.rs
index e8eaf33..c400cc1 100644
--- a/parquet/src/arrow/arrow_writer.rs
+++ b/parquet/src/arrow/arrow_writer.rs
@@ -92,6 +92,8 @@
         let mut row_group_writer = self.writer.next_row_group()?;
         for (array, field) in batch.columns().iter().zip(batch.schema().fields()) {
             let mut levels = batch_level.calculate_array_levels(array, field);
+            // Reverse levels as we pop() them when writing arrays
+            levels.reverse();
             write_leaves(&mut row_group_writer, array, &mut levels)?;
         }
 
@@ -741,7 +743,6 @@
     }
 
     #[test]
-    #[ignore = "See ARROW-11294, data is correct but list field name is incorrect"]
     fn arrow_writer_complex() {
         // define schema
         let struct_field_d = Field::new("d", DataType::Float64, true);
@@ -934,7 +935,6 @@
             let actual_data = actual_batch.column(i).data();
 
             assert_eq!(expected_data, actual_data);
-            // assert_eq!(expected_data, actual_data, "L: {:#?}\nR: {:#?}", expected_data, actual_data);
         }
     }