| <?xml version='1.0' encoding='UTF-8' standalone='no'?> |
| <doxygen xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:noNamespaceSchemaLocation="compound.xsd" version="1.9.8" xml:lang="en-US"> |
| <compounddef id="parquet__reader_8cc" kind="file" language="C++"> |
| <compoundname>parquet_reader.cc</compoundname> |
| <includes refid="parquet__reader_8h" local="yes">iceberg/parquet/parquet_reader.h</includes> |
| <includes local="no">numeric</includes> |
| <includes local="no">arrow/c/bridge.h</includes> |
| <includes local="no">arrow/memory_pool.h</includes> |
| <includes local="no">arrow/record_batch.h</includes> |
| <includes local="no">arrow/result.h</includes> |
| <includes local="no">arrow/type.h</includes> |
| <includes local="no">arrow/util/key_value_metadata.h</includes> |
| <includes local="no">parquet/arrow/reader.h</includes> |
| <includes local="no">parquet/arrow/schema.h</includes> |
| <includes local="no">parquet/file_reader.h</includes> |
| <includes local="no">parquet/properties.h</includes> |
| <includes refid="arrow__fs__file__io__internal_8h" local="yes">iceberg/arrow/arrow_fs_file_io_internal.h</includes> |
| <includes refid="arrow__status__internal_8h" local="yes">iceberg/arrow/arrow_status_internal.h</includes> |
| <includes refid="metadata__column__util__internal_8h" local="yes">iceberg/arrow/metadata_column_util_internal.h</includes> |
| <includes refid="parquet__data__util__internal_8h" local="yes">iceberg/parquet/parquet_data_util_internal.h</includes> |
| <includes refid="parquet__register_8h" local="yes">iceberg/parquet/parquet_register.h</includes> |
| <includes refid="parquet__schema__util__internal_8h" local="yes">iceberg/parquet/parquet_schema_util_internal.h</includes> |
| <includes refid="result_8h" local="yes">iceberg/result.h</includes> |
| <includes refid="schema__internal_8h" local="yes">iceberg/schema_internal.h</includes> |
| <includes refid="schema__util_8h" local="yes">iceberg/schema_util.h</includes> |
| <includes refid="checked__cast_8h" local="yes">iceberg/util/checked_cast.h</includes> |
| <includes refid="macros_8h" local="yes">iceberg/util/macros.h</includes> |
| <incdepgraph> |
| <node id="43"> |
| <label>iceberg/arrow/arrow_fs_file_io_internal.h</label> |
| <link refid="arrow__fs__file__io__internal_8h_source"/> |
| <childnode refid="5" relation="include"> |
| </childnode> |
| <childnode refid="44" relation="include"> |
| </childnode> |
| <childnode refid="45" relation="include"> |
| </childnode> |
| <childnode refid="31" relation="include"> |
| </childnode> |
| </node> |
| <node id="46"> |
| <label>iceberg/arrow/arrow_status_internal.h</label> |
| <link refid="arrow__status__internal_8h_source"/> |
| <childnode refid="36" relation="include"> |
| </childnode> |
| <childnode refid="47" relation="include"> |
| </childnode> |
| <childnode refid="13" relation="include"> |
| </childnode> |
| </node> |
| <node id="48"> |
| <label>iceberg/arrow/metadata_column_util_internal.h</label> |
| <link refid="metadata__column__util__internal_8h"/> |
| <childnode refid="8" relation="include"> |
| </childnode> |
| <childnode refid="5" relation="include"> |
| </childnode> |
| <childnode refid="16" relation="include"> |
| </childnode> |
| <childnode refid="49" relation="include"> |
| </childnode> |
| <childnode refid="13" relation="include"> |
| </childnode> |
| </node> |
| <node id="7"> |
| <label>iceberg/arrow_c_data.h</label> |
| <link refid="arrow__c__data_8h"/> |
| <childnode refid="8" relation="include"> |
| </childnode> |
| </node> |
| <node id="27"> |
| <label>iceberg/exception.h</label> |
| <link refid="exception_8h"/> |
| <childnode refid="28" relation="include"> |
| </childnode> |
| <childnode refid="12" relation="include"> |
| </childnode> |
| </node> |
| <node id="54"> |
| <label>iceberg/expression/literal.h</label> |
| <link refid="literal_8h_source"/> |
| <childnode refid="55" relation="include"> |
| </childnode> |
| <childnode refid="5" relation="include"> |
| </childnode> |
| <childnode refid="16" relation="include"> |
| </childnode> |
| <childnode refid="52" relation="include"> |
| </childnode> |
| <childnode refid="53" relation="include"> |
| </childnode> |
| <childnode refid="12" relation="include"> |
| </childnode> |
| <childnode refid="13" relation="include"> |
| </childnode> |
| <childnode refid="56" relation="include"> |
| </childnode> |
| <childnode refid="64" relation="include"> |
| </childnode> |
| <childnode refid="60" relation="include"> |
| </childnode> |
| <childnode refid="66" relation="include"> |
| </childnode> |
| <childnode refid="67" relation="include"> |
| </childnode> |
| </node> |
| <node id="9"> |
| <label>iceberg/file_format.h</label> |
| <link refid="file__format_8h"/> |
| <childnode refid="10" relation="include"> |
| </childnode> |
| <childnode refid="11" relation="include"> |
| </childnode> |
| <childnode refid="12" relation="include"> |
| </childnode> |
| <childnode refid="13" relation="include"> |
| </childnode> |
| <childnode refid="17" relation="include"> |
| </childnode> |
| </node> |
| <node id="45"> |
| <label>iceberg/file_io.h</label> |
| <link refid="file__io_8h_source"/> |
| <childnode refid="6" relation="include"> |
| </childnode> |
| <childnode refid="16" relation="include"> |
| </childnode> |
| <childnode refid="10" relation="include"> |
| </childnode> |
| <childnode refid="12" relation="include"> |
| </childnode> |
| <childnode refid="13" relation="include"> |
| </childnode> |
| </node> |
| <node id="3"> |
| <label>iceberg/file_reader.h</label> |
| <link refid="file__reader_8h"/> |
| <childnode refid="4" relation="include"> |
| </childnode> |
| <childnode refid="5" relation="include"> |
| </childnode> |
| <childnode refid="6" relation="include"> |
| </childnode> |
| <childnode refid="7" relation="include"> |
| </childnode> |
| <childnode refid="9" relation="include"> |
| </childnode> |
| <childnode refid="13" relation="include"> |
| </childnode> |
| <childnode refid="24" relation="include"> |
| </childnode> |
| <childnode refid="25" relation="include"> |
| </childnode> |
| </node> |
| <node id="31"> |
| <label>iceberg/iceberg_bundle_export.h</label> |
| <link refid="iceberg__bundle__export_8h_source"/> |
| </node> |
| <node id="12"> |
| <label>iceberg/iceberg_export.h</label> |
| <link refid="iceberg__export_8h_source"/> |
| </node> |
| <node id="50"> |
| <label>iceberg/parquet/parquet_data_util_internal.h</label> |
| <link refid="parquet__data__util__internal_8h_source"/> |
| <childnode refid="49" relation="include"> |
| </childnode> |
| <childnode refid="48" relation="include"> |
| </childnode> |
| <childnode refid="51" relation="include"> |
| </childnode> |
| </node> |
| <node id="1"> |
| <label>iceberg/parquet/parquet_reader.cc</label> |
| <link refid="parquet__reader_8cc"/> |
| <childnode refid="2" relation="include"> |
| </childnode> |
| <childnode refid="32" relation="include"> |
| </childnode> |
| <childnode refid="33" relation="include"> |
| </childnode> |
| <childnode refid="34" relation="include"> |
| </childnode> |
| <childnode refid="35" relation="include"> |
| </childnode> |
| <childnode refid="36" relation="include"> |
| </childnode> |
| <childnode refid="37" relation="include"> |
| </childnode> |
| <childnode refid="38" relation="include"> |
| </childnode> |
| <childnode refid="39" relation="include"> |
| </childnode> |
| <childnode refid="40" relation="include"> |
| </childnode> |
| <childnode refid="41" relation="include"> |
| </childnode> |
| <childnode refid="42" relation="include"> |
| </childnode> |
| <childnode refid="43" relation="include"> |
| </childnode> |
| <childnode refid="46" relation="include"> |
| </childnode> |
| <childnode refid="48" relation="include"> |
| </childnode> |
| <childnode refid="50" relation="include"> |
| </childnode> |
| <childnode refid="68" relation="include"> |
| </childnode> |
| <childnode refid="69" relation="include"> |
| </childnode> |
| <childnode refid="13" relation="include"> |
| </childnode> |
| <childnode refid="72" relation="include"> |
| </childnode> |
| <childnode refid="51" relation="include"> |
| </childnode> |
| <childnode refid="74" relation="include"> |
| </childnode> |
| <childnode refid="29" relation="include"> |
| </childnode> |
| </node> |
| <node id="2"> |
| <label>iceberg/parquet/parquet_reader.h</label> |
| <link refid="parquet__reader_8h_source"/> |
| <childnode refid="3" relation="include"> |
| </childnode> |
| <childnode refid="31" relation="include"> |
| </childnode> |
| </node> |
| <node id="68"> |
| <label>iceberg/parquet/parquet_register.h</label> |
| <link refid="parquet__register_8h"/> |
| <childnode refid="31" relation="include"> |
| </childnode> |
| </node> |
| <node id="69"> |
| <label>iceberg/parquet/parquet_schema_util_internal.h</label> |
| <link refid="parquet__schema__util__internal_8h_source"/> |
| <childnode refid="6" relation="include"> |
| </childnode> |
| <childnode refid="40" relation="include"> |
| </childnode> |
| <childnode refid="70" relation="include"> |
| </childnode> |
| <childnode refid="51" relation="include"> |
| </childnode> |
| </node> |
| <node id="13"> |
| <label>iceberg/result.h</label> |
| <link refid="result_8h_source"/> |
| <childnode refid="14" relation="include"> |
| </childnode> |
| <childnode refid="15" relation="include"> |
| </childnode> |
| <childnode refid="16" relation="include"> |
| </childnode> |
| <childnode refid="12" relation="include"> |
| </childnode> |
| </node> |
| <node id="70"> |
| <label>iceberg/schema.h</label> |
| <link refid="schema_8h"/> |
| <childnode refid="8" relation="include"> |
| </childnode> |
| <childnode refid="6" relation="include"> |
| </childnode> |
| <childnode refid="16" relation="include"> |
| </childnode> |
| <childnode refid="26" relation="include"> |
| </childnode> |
| <childnode refid="71" relation="include"> |
| </childnode> |
| <childnode refid="53" relation="include"> |
| </childnode> |
| <childnode refid="12" relation="include"> |
| </childnode> |
| <childnode refid="13" relation="include"> |
| </childnode> |
| <childnode refid="59" relation="include"> |
| </childnode> |
| <childnode refid="56" relation="include"> |
| </childnode> |
| <childnode refid="61" relation="include"> |
| </childnode> |
| <childnode refid="17" relation="include"> |
| </childnode> |
| </node> |
| <node id="59"> |
| <label>iceberg/schema_field.h</label> |
| <link refid="schema__field_8h"/> |
| <childnode refid="8" relation="include"> |
| </childnode> |
| <childnode refid="5" relation="include"> |
| </childnode> |
| <childnode refid="16" relation="include"> |
| </childnode> |
| <childnode refid="10" relation="include"> |
| </childnode> |
| <childnode refid="12" relation="include"> |
| </childnode> |
| <childnode refid="13" relation="include"> |
| </childnode> |
| <childnode refid="24" relation="include"> |
| </childnode> |
| <childnode refid="60" relation="include"> |
| </childnode> |
| </node> |
| <node id="72"> |
| <label>iceberg/schema_internal.h</label> |
| <link refid="schema__internal_8h_source"/> |
| <childnode refid="5" relation="include"> |
| </childnode> |
| <childnode refid="6" relation="include"> |
| </childnode> |
| <childnode refid="73" relation="include"> |
| </childnode> |
| <childnode refid="12" relation="include"> |
| </childnode> |
| <childnode refid="13" relation="include"> |
| </childnode> |
| <childnode refid="24" relation="include"> |
| </childnode> |
| </node> |
| <node id="51"> |
| <label>iceberg/schema_util.h</label> |
| <link refid="schema__util_8h_source"/> |
| <childnode refid="5" relation="include"> |
| </childnode> |
| <childnode refid="16" relation="include"> |
| </childnode> |
| <childnode refid="52" relation="include"> |
| </childnode> |
| <childnode refid="53" relation="include"> |
| </childnode> |
| <childnode refid="54" relation="include"> |
| </childnode> |
| <childnode refid="12" relation="include"> |
| </childnode> |
| <childnode refid="13" relation="include"> |
| </childnode> |
| <childnode refid="24" relation="include"> |
| </childnode> |
| </node> |
| <node id="56"> |
| <label>iceberg/type.h</label> |
| <link refid="type_8h"/> |
| <childnode refid="57" relation="include"> |
| </childnode> |
| <childnode refid="8" relation="include"> |
| </childnode> |
| <childnode refid="5" relation="include"> |
| </childnode> |
| <childnode refid="6" relation="include"> |
| </childnode> |
| <childnode refid="58" relation="include"> |
| </childnode> |
| <childnode refid="16" relation="include"> |
| </childnode> |
| <childnode refid="26" relation="include"> |
| </childnode> |
| <childnode refid="53" relation="include"> |
| </childnode> |
| <childnode refid="12" relation="include"> |
| </childnode> |
| <childnode refid="13" relation="include"> |
| </childnode> |
| <childnode refid="59" relation="include"> |
| </childnode> |
| <childnode refid="60" relation="include"> |
| </childnode> |
| <childnode refid="61" relation="include"> |
| </childnode> |
| </node> |
| <node id="24"> |
| <label>iceberg/type_fwd.h</label> |
| <link refid="type__fwd_8h"/> |
| </node> |
| <node id="74"> |
| <label>iceberg/util/checked_cast.h</label> |
| <link refid="checked__cast_8h"/> |
| <childnode refid="5" relation="include"> |
| </childnode> |
| <childnode refid="22" relation="include"> |
| </childnode> |
| <childnode refid="11" relation="include"> |
| </childnode> |
| </node> |
| <node id="25"> |
| <label>iceberg/util/config.h</label> |
| <link refid="config_8h_source"/> |
| <childnode refid="15" relation="include"> |
| </childnode> |
| <childnode refid="4" relation="include"> |
| </childnode> |
| <childnode refid="16" relation="include"> |
| </childnode> |
| <childnode refid="26" relation="include"> |
| </childnode> |
| <childnode refid="27" relation="include"> |
| </childnode> |
| <childnode refid="29" relation="include"> |
| </childnode> |
| <childnode refid="17" relation="include"> |
| </childnode> |
| </node> |
| <node id="64"> |
| <label>iceberg/util/decimal.h</label> |
| <link refid="decimal_8h"/> |
| <childnode refid="57" relation="include"> |
| </childnode> |
| <childnode refid="55" relation="include"> |
| </childnode> |
| <childnode refid="8" relation="include"> |
| </childnode> |
| <childnode refid="65" relation="include"> |
| </childnode> |
| <childnode refid="16" relation="include"> |
| </childnode> |
| <childnode refid="10" relation="include"> |
| </childnode> |
| <childnode refid="22" relation="include"> |
| </childnode> |
| <childnode refid="53" relation="include"> |
| </childnode> |
| <childnode refid="12" relation="include"> |
| </childnode> |
| <childnode refid="13" relation="include"> |
| </childnode> |
| <childnode refid="60" relation="include"> |
| </childnode> |
| <childnode refid="66" relation="include"> |
| </childnode> |
| </node> |
| <node id="60"> |
| <label>iceberg/util/formattable.h</label> |
| <link refid="formattable_8h"/> |
| <childnode refid="16" relation="include"> |
| </childnode> |
| <childnode refid="12" relation="include"> |
| </childnode> |
| </node> |
| <node id="66"> |
| <label>iceberg/util/int128.h</label> |
| <link refid="int128_8h"/> |
| </node> |
| <node id="61"> |
| <label>iceberg/util/lazy.h</label> |
| <link refid="lazy_8h"/> |
| <childnode refid="62" relation="include"> |
| </childnode> |
| <childnode refid="4" relation="include"> |
| </childnode> |
| <childnode refid="63" relation="include"> |
| </childnode> |
| <childnode refid="13" relation="include"> |
| </childnode> |
| <childnode refid="29" relation="include"> |
| </childnode> |
| </node> |
| <node id="29"> |
| <label>iceberg/util/macros.h</label> |
| <link refid="macros_8h_source"/> |
| <childnode refid="30" relation="include"> |
| </childnode> |
| <childnode refid="27" relation="include"> |
| </childnode> |
| <childnode refid="13" relation="include"> |
| </childnode> |
| </node> |
| <node id="17"> |
| <label>iceberg/util/string_util.h</label> |
| <link refid="string__util_8h_source"/> |
| <childnode refid="18" relation="include"> |
| </childnode> |
| <childnode refid="19" relation="include"> |
| </childnode> |
| <childnode refid="20" relation="include"> |
| </childnode> |
| <childnode refid="21" relation="include"> |
| </childnode> |
| <childnode refid="16" relation="include"> |
| </childnode> |
| <childnode refid="10" relation="include"> |
| </childnode> |
| <childnode refid="22" relation="include"> |
| </childnode> |
| <childnode refid="23" relation="include"> |
| </childnode> |
| <childnode refid="11" relation="include"> |
| </childnode> |
| <childnode refid="12" relation="include"> |
| </childnode> |
| <childnode refid="13" relation="include"> |
| </childnode> |
| </node> |
| <node id="67"> |
| <label>iceberg/util/uuid.h</label> |
| <link refid="uuid_8h"/> |
| <childnode refid="57" relation="include"> |
| </childnode> |
| <childnode refid="8" relation="include"> |
| </childnode> |
| <childnode refid="58" relation="include"> |
| </childnode> |
| <childnode refid="10" relation="include"> |
| </childnode> |
| <childnode refid="12" relation="include"> |
| </childnode> |
| <childnode refid="13" relation="include"> |
| </childnode> |
| <childnode refid="60" relation="include"> |
| </childnode> |
| </node> |
| <node id="18"> |
| <label>algorithm</label> |
| </node> |
| <node id="57"> |
| <label>array</label> |
| </node> |
| <node id="33"> |
| <label>arrow/c/bridge.h</label> |
| </node> |
| <node id="44"> |
| <label>arrow/filesystem/filesystem.h</label> |
| </node> |
| <node id="34"> |
| <label>arrow/memory_pool.h</label> |
| </node> |
| <node id="35"> |
| <label>arrow/record_batch.h</label> |
| </node> |
| <node id="36"> |
| <label>arrow/result.h</label> |
| </node> |
| <node id="47"> |
| <label>arrow/status.h</label> |
| </node> |
| <node id="37"> |
| <label>arrow/type.h</label> |
| </node> |
| <node id="49"> |
| <label>arrow/type_fwd.h</label> |
| </node> |
| <node id="38"> |
| <label>arrow/util/key_value_metadata.h</label> |
| </node> |
| <node id="30"> |
| <label>cassert</label> |
| </node> |
| <node id="19"> |
| <label>cerrno</label> |
| </node> |
| <node id="20"> |
| <label>charconv</label> |
| </node> |
| <node id="55"> |
| <label>compare</label> |
| </node> |
| <node id="62"> |
| <label>concepts</label> |
| </node> |
| <node id="8"> |
| <label>cstdint</label> |
| </node> |
| <node id="14"> |
| <label>expected</label> |
| </node> |
| <node id="15"> |
| <label>format</label> |
| </node> |
| <node id="4"> |
| <label>functional</label> |
| </node> |
| <node id="65"> |
| <label>iosfwd</label> |
| </node> |
| <node id="5"> |
| <label>memory</label> |
| </node> |
| <node id="63"> |
| <label>mutex</label> |
| </node> |
| <node id="73"> |
| <label>nanoarrow/nanoarrow.h</label> |
| </node> |
| <node id="32"> |
| <label>numeric</label> |
| </node> |
| <node id="6"> |
| <label>optional</label> |
| </node> |
| <node id="39"> |
| <label>parquet/arrow/reader.h</label> |
| </node> |
| <node id="40"> |
| <label>parquet/arrow/schema.h</label> |
| </node> |
| <node id="41"> |
| <label>parquet/file_reader.h</label> |
| </node> |
| <node id="42"> |
| <label>parquet/properties.h</label> |
| </node> |
| <node id="21"> |
| <label>ranges</label> |
| </node> |
| <node id="58"> |
| <label>span</label> |
| </node> |
| <node id="28"> |
| <label>stdexcept</label> |
| </node> |
| <node id="16"> |
| <label>string</label> |
| </node> |
| <node id="10"> |
| <label>string_view</label> |
| </node> |
| <node id="22"> |
| <label>type_traits</label> |
| </node> |
| <node id="23"> |
| <label>typeinfo</label> |
| </node> |
| <node id="26"> |
| <label>unordered_map</label> |
| </node> |
| <node id="71"> |
| <label>unordered_set</label> |
| </node> |
| <node id="11"> |
| <label>utility</label> |
| </node> |
| <node id="52"> |
| <label>variant</label> |
| </node> |
| <node id="53"> |
| <label>vector</label> |
| </node> |
| </incdepgraph> |
| <innerclass refid="structiceberg_1_1parquet_1_1ReadContext" prot="public">iceberg::parquet::ReadContext</innerclass> |
| <innerclass refid="classiceberg_1_1parquet_1_1ParquetReader_1_1Impl" prot="public">iceberg::parquet::ParquetReader::Impl</innerclass> |
| <innernamespace refid="namespaceiceberg">iceberg</innernamespace> |
| <innernamespace refid="namespaceiceberg_1_1parquet">iceberg::parquet</innernamespace> |
| <briefdescription> |
| </briefdescription> |
| <detaileddescription> |
| </detaileddescription> |
| <programlisting> |
| <codeline lineno="1"><highlight class="comment">/*</highlight></codeline> |
| <codeline lineno="2"><highlight class="comment"><sp/>*<sp/>Licensed<sp/>to<sp/>the<sp/>Apache<sp/>Software<sp/>Foundation<sp/>(ASF)<sp/>under<sp/>one</highlight></codeline> |
| <codeline lineno="3"><highlight class="comment"><sp/>*<sp/>or<sp/>more<sp/>contributor<sp/>license<sp/>agreements.<sp/><sp/>See<sp/>the<sp/>NOTICE<sp/>file</highlight></codeline> |
| <codeline lineno="4"><highlight class="comment"><sp/>*<sp/>distributed<sp/>with<sp/>this<sp/>work<sp/>for<sp/>additional<sp/>information</highlight></codeline> |
| <codeline lineno="5"><highlight class="comment"><sp/>*<sp/>regarding<sp/>copyright<sp/>ownership.<sp/><sp/>The<sp/>ASF<sp/>licenses<sp/>this<sp/>file</highlight></codeline> |
| <codeline lineno="6"><highlight class="comment"><sp/>*<sp/>to<sp/>you<sp/>under<sp/>the<sp/>Apache<sp/>License,<sp/>Version<sp/>2.0<sp/>(the</highlight></codeline> |
| <codeline lineno="7"><highlight class="comment"><sp/>*<sp/>"License");<sp/>you<sp/>may<sp/>not<sp/>use<sp/>this<sp/>file<sp/>except<sp/>in<sp/>compliance</highlight></codeline> |
| <codeline lineno="8"><highlight class="comment"><sp/>*<sp/>with<sp/>the<sp/>License.<sp/><sp/>You<sp/>may<sp/>obtain<sp/>a<sp/>copy<sp/>of<sp/>the<sp/>License<sp/>at</highlight></codeline> |
| <codeline lineno="9"><highlight class="comment"><sp/>*</highlight></codeline> |
| <codeline lineno="10"><highlight class="comment"><sp/>*<sp/><sp/><sp/>http://www.apache.org/licenses/LICENSE-2.0</highlight></codeline> |
| <codeline lineno="11"><highlight class="comment"><sp/>*</highlight></codeline> |
| <codeline lineno="12"><highlight class="comment"><sp/>*<sp/>Unless<sp/>required<sp/>by<sp/>applicable<sp/>law<sp/>or<sp/>agreed<sp/>to<sp/>in<sp/>writing,</highlight></codeline> |
| <codeline lineno="13"><highlight class="comment"><sp/>*<sp/>software<sp/>distributed<sp/>under<sp/>the<sp/>License<sp/>is<sp/>distributed<sp/>on<sp/>an</highlight></codeline> |
| <codeline lineno="14"><highlight class="comment"><sp/>*<sp/>"AS<sp/>IS"<sp/>BASIS,<sp/>WITHOUT<sp/>WARRANTIES<sp/>OR<sp/>CONDITIONS<sp/>OF<sp/>ANY</highlight></codeline> |
| <codeline lineno="15"><highlight class="comment"><sp/>*<sp/>KIND,<sp/>either<sp/>express<sp/>or<sp/>implied.<sp/><sp/>See<sp/>the<sp/>License<sp/>for<sp/>the</highlight></codeline> |
| <codeline lineno="16"><highlight class="comment"><sp/>*<sp/>specific<sp/>language<sp/>governing<sp/>permissions<sp/>and<sp/>limitations</highlight></codeline> |
| <codeline lineno="17"><highlight class="comment"><sp/>*<sp/>under<sp/>the<sp/>License.</highlight></codeline> |
| <codeline lineno="18"><highlight class="comment"><sp/>*/</highlight><highlight class="normal"></highlight></codeline> |
| <codeline lineno="19"><highlight class="normal"></highlight></codeline> |
| <codeline lineno="20"><highlight class="normal"></highlight><highlight class="preprocessor">#include<sp/>"iceberg/parquet/parquet_reader.h"</highlight><highlight class="normal"></highlight></codeline> |
| <codeline lineno="21"><highlight class="normal"></highlight></codeline> |
| <codeline lineno="22"><highlight class="normal"></highlight><highlight class="preprocessor">#include<sp/><numeric></highlight><highlight class="normal"></highlight></codeline> |
| <codeline lineno="23"><highlight class="normal"></highlight></codeline> |
| <codeline lineno="24"><highlight class="normal"></highlight><highlight class="preprocessor">#include<sp/><arrow/c/bridge.h></highlight><highlight class="normal"></highlight></codeline> |
| <codeline lineno="25"><highlight class="normal"></highlight><highlight class="preprocessor">#include<sp/><arrow/memory_pool.h></highlight><highlight class="normal"></highlight></codeline> |
| <codeline lineno="26"><highlight class="normal"></highlight><highlight class="preprocessor">#include<sp/><arrow/record_batch.h></highlight><highlight class="normal"></highlight></codeline> |
| <codeline lineno="27"><highlight class="normal"></highlight><highlight class="preprocessor">#include<sp/><arrow/result.h></highlight><highlight class="normal"></highlight></codeline> |
| <codeline lineno="28"><highlight class="normal"></highlight><highlight class="preprocessor">#include<sp/><arrow/type.h></highlight><highlight class="normal"></highlight></codeline> |
| <codeline lineno="29"><highlight class="normal"></highlight><highlight class="preprocessor">#include<sp/><arrow/util/key_value_metadata.h></highlight><highlight class="normal"></highlight></codeline> |
| <codeline lineno="30"><highlight class="normal"></highlight><highlight class="preprocessor">#include<sp/><parquet/arrow/reader.h></highlight><highlight class="normal"></highlight></codeline> |
| <codeline lineno="31"><highlight class="normal"></highlight><highlight class="preprocessor">#include<sp/><parquet/arrow/schema.h></highlight><highlight class="normal"></highlight></codeline> |
| <codeline lineno="32"><highlight class="normal"></highlight><highlight class="preprocessor">#include<sp/><parquet/file_reader.h></highlight><highlight class="normal"></highlight></codeline> |
| <codeline lineno="33"><highlight class="normal"></highlight><highlight class="preprocessor">#include<sp/><parquet/properties.h></highlight><highlight class="normal"></highlight></codeline> |
| <codeline lineno="34"><highlight class="normal"></highlight></codeline> |
| <codeline lineno="35"><highlight class="normal"></highlight><highlight class="preprocessor">#include<sp/>"iceberg/arrow/arrow_fs_file_io_internal.h"</highlight><highlight class="normal"></highlight></codeline> |
| <codeline lineno="36"><highlight class="normal"></highlight><highlight class="preprocessor">#include<sp/>"iceberg/arrow/arrow_status_internal.h"</highlight><highlight class="normal"></highlight></codeline> |
| <codeline lineno="37"><highlight class="normal"></highlight><highlight class="preprocessor">#include<sp/>"<ref refid="metadata__column__util__internal_8h" kindref="compound">iceberg/arrow/metadata_column_util_internal.h</ref>"</highlight><highlight class="normal"></highlight></codeline> |
| <codeline lineno="38"><highlight class="normal"></highlight><highlight class="preprocessor">#include<sp/>"iceberg/parquet/parquet_data_util_internal.h"</highlight><highlight class="normal"></highlight></codeline> |
| <codeline lineno="39"><highlight class="normal"></highlight><highlight class="preprocessor">#include<sp/>"<ref refid="parquet__register_8h" kindref="compound">iceberg/parquet/parquet_register.h</ref>"</highlight><highlight class="normal"></highlight></codeline> |
| <codeline lineno="40"><highlight class="normal"></highlight><highlight class="preprocessor">#include<sp/>"iceberg/parquet/parquet_schema_util_internal.h"</highlight><highlight class="normal"></highlight></codeline> |
| <codeline lineno="41"><highlight class="normal"></highlight><highlight class="preprocessor">#include<sp/>"iceberg/result.h"</highlight><highlight class="normal"></highlight></codeline> |
| <codeline lineno="42"><highlight class="normal"></highlight><highlight class="preprocessor">#include<sp/>"iceberg/schema_internal.h"</highlight><highlight class="normal"></highlight></codeline> |
| <codeline lineno="43"><highlight class="normal"></highlight><highlight class="preprocessor">#include<sp/>"iceberg/schema_util.h"</highlight><highlight class="normal"></highlight></codeline> |
| <codeline lineno="44"><highlight class="normal"></highlight><highlight class="preprocessor">#include<sp/>"<ref refid="checked__cast_8h" kindref="compound">iceberg/util/checked_cast.h</ref>"</highlight><highlight class="normal"></highlight></codeline> |
| <codeline lineno="45"><highlight class="normal"></highlight><highlight class="preprocessor">#include<sp/>"iceberg/util/macros.h"</highlight><highlight class="normal"></highlight></codeline> |
| <codeline lineno="46"><highlight class="normal"></highlight></codeline> |
| <codeline lineno="47"><highlight class="normal"></highlight><highlight class="keyword">namespace<sp/></highlight><highlight class="normal">iceberg::parquet<sp/>{</highlight></codeline> |
| <codeline lineno="48"><highlight class="normal"></highlight></codeline> |
| <codeline lineno="49"><highlight class="normal"></highlight><highlight class="keyword">namespace<sp/></highlight><highlight class="normal">{</highlight></codeline> |
| <codeline lineno="50"><highlight class="normal"></highlight></codeline> |
| <codeline lineno="51"><highlight class="normal">Result<std::shared_ptr<::arrow::io::RandomAccessFile>><sp/>OpenInputStream(</highlight></codeline> |
| <codeline lineno="52"><highlight class="normal"><sp/><sp/><sp/><sp/></highlight><highlight class="keyword">const</highlight><highlight class="normal"><sp/>ReaderOptions&<sp/>options)<sp/>{</highlight></codeline> |
| <codeline lineno="53"><highlight class="normal"><sp/><sp/>::arrow::fs::FileInfo<sp/>file_info(options.path,<sp/>::arrow::fs::FileType::File);</highlight></codeline> |
| <codeline lineno="54"><highlight class="normal"><sp/><sp/></highlight><highlight class="keywordflow">if</highlight><highlight class="normal"><sp/>(options.length)<sp/>{</highlight></codeline> |
| <codeline lineno="55"><highlight class="normal"><sp/><sp/><sp/><sp/>file_info.set_size(options.length.value());</highlight></codeline> |
| <codeline lineno="56"><highlight class="normal"><sp/><sp/>}</highlight></codeline> |
| <codeline lineno="57"><highlight class="normal"></highlight></codeline> |
| <codeline lineno="58"><highlight class="normal"><sp/><sp/></highlight><highlight class="keyword">auto</highlight><highlight class="normal"><sp/>io<sp/>=<sp/>internal::checked_pointer_cast<arrow::ArrowFileSystemFileIO>(options.io);</highlight></codeline> |
| <codeline lineno="59"><highlight class="normal"><sp/><sp/>ICEBERG_ARROW_ASSIGN_OR_RETURN(</highlight><highlight class="keyword">auto</highlight><highlight class="normal"><sp/>input,<sp/>io->fs()->OpenInputFile(file_info));</highlight></codeline> |
| <codeline lineno="60"><highlight class="normal"><sp/><sp/></highlight><highlight class="keywordflow">return</highlight><highlight class="normal"><sp/>input;</highlight></codeline> |
| <codeline lineno="61"><highlight class="normal">}</highlight></codeline> |
| <codeline lineno="62"><highlight class="normal"></highlight></codeline> |
| <codeline lineno="63"><highlight class="normal">Result<SchemaProjection><sp/>BuildProjection(::parquet::arrow::FileReader*<sp/>reader,</highlight></codeline> |
| <codeline lineno="64"><highlight class="normal"><sp/><sp/><sp/><sp/><sp/><sp/><sp/><sp/><sp/><sp/><sp/><sp/><sp/><sp/><sp/><sp/><sp/><sp/><sp/><sp/><sp/><sp/><sp/><sp/><sp/><sp/><sp/><sp/><sp/><sp/><sp/><sp/><sp/><sp/><sp/><sp/><sp/><sp/><sp/><sp/><sp/></highlight><highlight class="keyword">const</highlight><highlight class="normal"><sp/>Schema&<sp/>read_schema)<sp/>{</highlight></codeline> |
| <codeline lineno="65"><highlight class="normal"><sp/><sp/></highlight><highlight class="keyword">auto</highlight><highlight class="normal"><sp/>metadata<sp/>=<sp/>reader->parquet_reader()->metadata();</highlight></codeline> |
| <codeline lineno="66"><highlight class="normal"></highlight></codeline> |
| <codeline lineno="67"><highlight class="normal"><sp/><sp/></highlight><highlight class="keywordflow">if</highlight><highlight class="normal"><sp/>(!HasFieldIds(metadata->schema()->schema_root()))<sp/>{</highlight></codeline> |
| <codeline lineno="68"><highlight class="normal"><sp/><sp/><sp/><sp/></highlight><highlight class="comment">//<sp/>TODO(gangwu):<sp/>apply<sp/>name<sp/>mapping<sp/>to<sp/>Parquet<sp/>schema</highlight><highlight class="normal"></highlight></codeline> |
| <codeline lineno="69"><highlight class="normal"><sp/><sp/><sp/><sp/></highlight><highlight class="keywordflow">return</highlight><highlight class="normal"><sp/>NotImplemented(</highlight><highlight class="stringliteral">"Applying<sp/>name<sp/>mapping<sp/>to<sp/>Parquet<sp/>schema<sp/>is<sp/>not<sp/>implemented"</highlight><highlight class="normal">);</highlight></codeline> |
| <codeline lineno="70"><highlight class="normal"><sp/><sp/>}</highlight></codeline> |
| <codeline lineno="71"><highlight class="normal"></highlight></codeline> |
| <codeline lineno="72"><highlight class="normal"><sp/><sp/>::parquet::arrow::SchemaManifest<sp/>schema_manifest;</highlight></codeline> |
| <codeline lineno="73"><highlight class="normal"><sp/><sp/>ICEBERG_ARROW_RETURN_NOT_OK(::parquet::arrow::SchemaManifest::Make(</highlight></codeline> |
| <codeline lineno="74"><highlight class="normal"><sp/><sp/><sp/><sp/><sp/><sp/>metadata->schema(),<sp/>metadata->key_value_metadata(),<sp/>reader->properties(),</highlight></codeline> |
| <codeline lineno="75"><highlight class="normal"><sp/><sp/><sp/><sp/><sp/><sp/>&schema_manifest));</highlight></codeline> |
| <codeline lineno="76"><highlight class="normal"></highlight></codeline> |
| <codeline lineno="77"><highlight class="normal"><sp/><sp/></highlight><highlight class="comment">//<sp/>Leverage<sp/>SchemaManifest<sp/>to<sp/>project<sp/>the<sp/>schema</highlight><highlight class="normal"></highlight></codeline> |
| <codeline lineno="78"><highlight class="normal"><sp/><sp/>ICEBERG_ASSIGN_OR_RAISE(</highlight><highlight class="keyword">auto</highlight><highlight class="normal"><sp/>projection,<sp/>Project(read_schema,<sp/>schema_manifest));</highlight></codeline> |
| <codeline lineno="79"><highlight class="normal"><sp/><sp/></highlight><highlight class="keywordflow">return</highlight><highlight class="normal"><sp/>projection;</highlight></codeline> |
| <codeline lineno="80"><highlight class="normal">}</highlight></codeline> |
| <codeline lineno="81"><highlight class="normal"></highlight></codeline> |
| <codeline lineno="82"><highlight class="normal"></highlight><highlight class="keyword">class<sp/></highlight><highlight class="normal">EmptyRecordBatchReader<sp/>:<sp/></highlight><highlight class="keyword">public</highlight><highlight class="normal"><sp/>::arrow::RecordBatchReader<sp/>{</highlight></codeline> |
| <codeline lineno="83"><highlight class="normal"><sp/></highlight><highlight class="keyword">public</highlight><highlight class="normal">:</highlight></codeline> |
| <codeline lineno="84"><highlight class="normal"><sp/><sp/>EmptyRecordBatchReader()<sp/>=<sp/></highlight><highlight class="keywordflow">default</highlight><highlight class="normal">;</highlight></codeline> |
| <codeline lineno="85"><highlight class="normal"><sp/><sp/>~EmptyRecordBatchReader()<sp/></highlight><highlight class="keyword">override</highlight><highlight class="normal"><sp/>=<sp/></highlight><highlight class="keywordflow">default</highlight><highlight class="normal">;</highlight></codeline> |
| <codeline lineno="86"><highlight class="normal"></highlight></codeline> |
| <codeline lineno="87"><highlight class="normal"><sp/><sp/>std::shared_ptr<::arrow::Schema><sp/>schema()</highlight><highlight class="keyword"><sp/>const<sp/>override<sp/></highlight><highlight class="normal">{<sp/></highlight><highlight class="keywordflow">return</highlight><highlight class="normal"><sp/></highlight><highlight class="keyword">nullptr</highlight><highlight class="normal">;<sp/>}</highlight></codeline> |
| <codeline lineno="88"><highlight class="normal"></highlight></codeline> |
| <codeline lineno="89"><highlight class="normal"><sp/><sp/>::arrow::Status<sp/>ReadNext(std::shared_ptr<::arrow::RecordBatch>*<sp/>batch)</highlight><highlight class="keyword"><sp/>override<sp/></highlight><highlight class="normal">{</highlight></codeline> |
| <codeline lineno="90"><highlight class="normal"><sp/><sp/><sp/><sp/>*batch<sp/>=<sp/></highlight><highlight class="keyword">nullptr</highlight><highlight class="normal">;</highlight></codeline> |
| <codeline lineno="91"><highlight class="normal"><sp/><sp/><sp/><sp/>return<sp/>::arrow::Status::OK();</highlight></codeline> |
| <codeline lineno="92"><highlight class="normal"><sp/><sp/>}</highlight></codeline> |
| <codeline lineno="93"><highlight class="normal">};</highlight></codeline> |
| <codeline lineno="94"><highlight class="normal"></highlight></codeline> |
| <codeline lineno="95"><highlight class="normal">}<sp/><sp/></highlight><highlight class="comment">//<sp/>namespace</highlight><highlight class="normal"></highlight></codeline> |
| <codeline lineno="96"><highlight class="normal"></highlight></codeline> |
| <codeline lineno="97"><highlight class="normal"></highlight><highlight class="comment">//<sp/>A<sp/>stateful<sp/>context<sp/>to<sp/>keep<sp/>track<sp/>of<sp/>the<sp/>reading<sp/>progress.</highlight><highlight class="normal"></highlight></codeline> |
| <codeline lineno="98" refid="structiceberg_1_1parquet_1_1ReadContext" refkind="compound"><highlight class="normal"></highlight><highlight class="keyword">struct<sp/></highlight><highlight class="normal"><ref refid="structiceberg_1_1parquet_1_1ReadContext" kindref="compound">ReadContext</ref><sp/>{</highlight></codeline> |
| <codeline lineno="99"><highlight class="normal"><sp/><sp/></highlight><highlight class="comment">//<sp/>The<sp/>arrow<sp/>schema<sp/>to<sp/>output<sp/>record<sp/>batches.<sp/>It<sp/>may<sp/>be<sp/>different<sp/>with</highlight><highlight class="normal"></highlight></codeline> |
| <codeline lineno="100"><highlight class="normal"><sp/><sp/></highlight><highlight class="comment">//<sp/>the<sp/>schema<sp/>of<sp/>record<sp/>batches<sp/>returned<sp/>by<sp/>`record_batch_reader_`</highlight><highlight class="normal"></highlight></codeline> |
| <codeline lineno="101"><highlight class="normal"><sp/><sp/></highlight><highlight class="comment">//<sp/>when<sp/>there<sp/>is<sp/>any<sp/>schema<sp/>evolution.</highlight><highlight class="normal"></highlight></codeline> |
| <codeline lineno="102"><highlight class="normal"><sp/><sp/>std::shared_ptr<::arrow::Schema><sp/>output_arrow_schema_;</highlight></codeline> |
| <codeline lineno="103"><highlight class="normal"><sp/><sp/></highlight><highlight class="comment">//<sp/>The<sp/>reader<sp/>to<sp/>read<sp/>record<sp/>batches<sp/>from<sp/>the<sp/>Parquet<sp/>file.</highlight><highlight class="normal"></highlight></codeline> |
| <codeline lineno="104"><highlight class="normal"><sp/><sp/>std::unique_ptr<::arrow::RecordBatchReader><sp/>record_batch_reader_;</highlight></codeline> |
| <codeline lineno="105"><highlight class="normal">};</highlight></codeline> |
| <codeline lineno="106"><highlight class="normal"></highlight></codeline> |
| <codeline lineno="107"><highlight class="normal"></highlight><highlight class="comment">//<sp/>TODO(gangwu):<sp/>list<sp/>of<sp/>work<sp/>items</highlight><highlight class="normal"></highlight></codeline> |
| <codeline lineno="108"><highlight class="normal"></highlight><highlight class="comment">//<sp/>1.<sp/>Make<sp/>the<sp/>memory<sp/>pool<sp/>configurable</highlight><highlight class="normal"></highlight></codeline> |
| <codeline lineno="109"><highlight class="normal"></highlight><highlight class="comment">//<sp/>2.<sp/>Catch<sp/>ParquetException<sp/>and<sp/>convert<sp/>to<sp/>Status/Result</highlight><highlight class="normal"></highlight></codeline> |
| <codeline lineno="110"><highlight class="normal"></highlight><highlight class="comment">//<sp/>3.<sp/>Add<sp/>utility<sp/>to<sp/>convert<sp/>Arrow<sp/>Status/Result<sp/>to<sp/>Iceberg<sp/>Status/Result</highlight><highlight class="normal"></highlight></codeline> |
| <codeline lineno="111"><highlight class="normal"></highlight><highlight class="comment">//<sp/>4.<sp/>Check<sp/>field<sp/>ids<sp/>and<sp/>apply<sp/>name<sp/>mapping<sp/>if<sp/>needed</highlight><highlight class="normal"></highlight></codeline> |
| <codeline lineno="112" refid="classiceberg_1_1parquet_1_1ParquetReader_1_1Impl" refkind="compound"><highlight class="normal"></highlight><highlight class="keyword">class<sp/></highlight><highlight class="normal"><ref refid="classiceberg_1_1parquet_1_1ParquetReader_1_1Impl" kindref="compound">ParquetReader::Impl</ref><sp/>{</highlight></codeline> |
| <codeline lineno="113"><highlight class="normal"><sp/></highlight><highlight class="keyword">public</highlight><highlight class="normal">:</highlight></codeline> |
| <codeline lineno="114"><highlight class="normal"><sp/><sp/></highlight><highlight class="comment">//<sp/>Open<sp/>the<sp/>Parquet<sp/>reader<sp/>with<sp/>the<sp/>given<sp/>options</highlight><highlight class="normal"></highlight></codeline> |
| <codeline lineno="115"><highlight class="normal"><sp/><sp/>Status<sp/>Open(</highlight><highlight class="keyword">const</highlight><highlight class="normal"><sp/><ref refid="structiceberg_1_1ReaderOptions" kindref="compound">ReaderOptions</ref>&<sp/>options)<sp/>{</highlight></codeline> |
| <codeline lineno="116"><highlight class="normal"><sp/><sp/><sp/><sp/></highlight><highlight class="keywordflow">if</highlight><highlight class="normal"><sp/>(options.<ref refid="structiceberg_1_1ReaderOptions_1a418c6556965b0db7f12131e4f0070304" kindref="member">projection</ref><sp/>==<sp/></highlight><highlight class="keyword">nullptr</highlight><highlight class="normal">)<sp/>{</highlight></codeline> |
| <codeline lineno="117"><highlight class="normal"><sp/><sp/><sp/><sp/><sp/><sp/></highlight><highlight class="keywordflow">return</highlight><highlight class="normal"><sp/>InvalidArgument(</highlight><highlight class="stringliteral">"Projected<sp/>schema<sp/>is<sp/>required<sp/>by<sp/>Parquet<sp/>reader"</highlight><highlight class="normal">);</highlight></codeline> |
| <codeline lineno="118"><highlight class="normal"><sp/><sp/><sp/><sp/>}</highlight></codeline> |
| <codeline lineno="119"><highlight class="normal"></highlight></codeline> |
| <codeline lineno="120"><highlight class="normal"><sp/><sp/><sp/><sp/>split_<sp/>=<sp/>options.<ref refid="structiceberg_1_1ReaderOptions_1ad23066ad0bd76b9bddf201e5b9bbeb0e" kindref="member">split</ref>;</highlight></codeline> |
| <codeline lineno="121"><highlight class="normal"><sp/><sp/><sp/><sp/>read_schema_<sp/>=<sp/>options.<ref refid="structiceberg_1_1ReaderOptions_1a418c6556965b0db7f12131e4f0070304" kindref="member">projection</ref>;</highlight></codeline> |
| <codeline lineno="122"><highlight class="normal"></highlight></codeline> |
| <codeline lineno="123"><highlight class="normal"><sp/><sp/><sp/><sp/></highlight><highlight class="comment">//<sp/>Prepare<sp/>reader<sp/>properties</highlight><highlight class="normal"></highlight></codeline> |
| <codeline lineno="124"><highlight class="normal"><sp/><sp/><sp/><sp/>::parquet::ReaderProperties<sp/>reader_properties(pool_);</highlight></codeline> |
| <codeline lineno="125"><highlight class="normal"><sp/><sp/><sp/><sp/>::parquet::ArrowReaderProperties<sp/>arrow_reader_properties;</highlight></codeline> |
| <codeline lineno="126"><highlight class="normal"><sp/><sp/><sp/><sp/>arrow_reader_properties.set_batch_size(</highlight></codeline> |
| <codeline lineno="127"><highlight class="normal"><sp/><sp/><sp/><sp/><sp/><sp/><sp/><sp/>options.<ref refid="structiceberg_1_1ReaderOptions_1abfe412d377c1207df04c3a2c4fe23d55" kindref="member">properties</ref>.Get(<ref refid="classiceberg_1_1ReaderProperties_1ac41dca455600b179913525bb79b2cd60" kindref="member">ReaderProperties::kBatchSize</ref>));</highlight></codeline> |
| <codeline lineno="128"><highlight class="normal"><sp/><sp/><sp/><sp/>arrow_reader_properties.set_arrow_extensions_enabled(</highlight><highlight class="keyword">true</highlight><highlight class="normal">);</highlight></codeline> |
| <codeline lineno="129"><highlight class="normal"></highlight></codeline> |
| <codeline lineno="130"><highlight class="normal"><sp/><sp/><sp/><sp/></highlight><highlight class="comment">//<sp/>Open<sp/>the<sp/>Parquet<sp/>file<sp/>reader</highlight><highlight class="normal"></highlight></codeline> |
| <codeline lineno="131"><highlight class="normal"><sp/><sp/><sp/><sp/>ICEBERG_ASSIGN_OR_RAISE(input_stream_,<sp/>OpenInputStream(options));</highlight></codeline> |
| <codeline lineno="132"><highlight class="normal"><sp/><sp/><sp/><sp/></highlight><highlight class="keyword">auto</highlight><highlight class="normal"><sp/>file_reader<sp/>=</highlight></codeline> |
| <codeline lineno="133"><highlight class="normal"><sp/><sp/><sp/><sp/><sp/><sp/><sp/><sp/>::parquet::ParquetFileReader::Open(input_stream_,<sp/>reader_properties);</highlight></codeline> |
| <codeline lineno="134"><highlight class="normal"><sp/><sp/><sp/><sp/>ICEBERG_ARROW_RETURN_NOT_OK(::parquet::arrow::FileReader::Make(</highlight></codeline> |
| <codeline lineno="135"><highlight class="normal"><sp/><sp/><sp/><sp/><sp/><sp/><sp/><sp/>pool_,<sp/>std::move(file_reader),<sp/>arrow_reader_properties,<sp/>&reader_));</highlight></codeline> |
| <codeline lineno="136"><highlight class="normal"></highlight></codeline> |
| <codeline lineno="137"><highlight class="normal"><sp/><sp/><sp/><sp/></highlight><highlight class="comment">//<sp/>Project<sp/>read<sp/>schema<sp/>onto<sp/>the<sp/>Parquet<sp/>file<sp/>schema</highlight><highlight class="normal"></highlight></codeline> |
| <codeline lineno="138"><highlight class="normal"><sp/><sp/><sp/><sp/>ICEBERG_ASSIGN_OR_RAISE(projection_,<sp/>BuildProjection(reader_.get(),<sp/>*read_schema_));</highlight></codeline> |
| <codeline lineno="139"><highlight class="normal"><sp/><sp/><sp/><sp/>metadata_context_<sp/>=<sp/>{.file_path<sp/>=<sp/>options.<ref refid="structiceberg_1_1ReaderOptions_1a959dc0c774f0a3d55991f1eb73753b2c" kindref="member">path</ref>,<sp/>.next_file_pos<sp/>=<sp/>0};</highlight></codeline> |
| <codeline lineno="140"><highlight class="normal"></highlight></codeline> |
| <codeline lineno="141"><highlight class="normal"><sp/><sp/><sp/><sp/></highlight><highlight class="keywordflow">return</highlight><highlight class="normal"><sp/>{};</highlight></codeline> |
| <codeline lineno="142"><highlight class="normal"><sp/><sp/>}</highlight></codeline> |
| <codeline lineno="143"><highlight class="normal"></highlight></codeline> |
| <codeline lineno="144"><highlight class="normal"><sp/><sp/></highlight><highlight class="comment">//<sp/>Read<sp/>the<sp/>next<sp/>batch<sp/>of<sp/>data</highlight><highlight class="normal"></highlight></codeline> |
| <codeline lineno="145"><highlight class="normal"><sp/><sp/>Result<std::optional<ArrowArray>><sp/>Next()<sp/>{</highlight></codeline> |
| <codeline lineno="146"><highlight class="normal"><sp/><sp/><sp/><sp/></highlight><highlight class="keywordflow">if</highlight><highlight class="normal"><sp/>(!context_)<sp/>{</highlight></codeline> |
| <codeline lineno="147"><highlight class="normal"><sp/><sp/><sp/><sp/><sp/><sp/>ICEBERG_RETURN_UNEXPECTED(InitReadContext());</highlight></codeline> |
| <codeline lineno="148"><highlight class="normal"><sp/><sp/><sp/><sp/>}</highlight></codeline> |
| <codeline lineno="149"><highlight class="normal"></highlight></codeline> |
| <codeline lineno="150"><highlight class="normal"><sp/><sp/><sp/><sp/>ICEBERG_ARROW_ASSIGN_OR_RETURN(</highlight><highlight class="keyword">auto</highlight><highlight class="normal"><sp/>batch,<sp/>context_->record_batch_reader_->Next());</highlight></codeline> |
| <codeline lineno="151"><highlight class="normal"><sp/><sp/><sp/><sp/></highlight><highlight class="keywordflow">if</highlight><highlight class="normal"><sp/>(!batch)<sp/>{</highlight></codeline> |
| <codeline lineno="152"><highlight class="normal"><sp/><sp/><sp/><sp/><sp/><sp/></highlight><highlight class="keywordflow">return</highlight><highlight class="normal"><sp/>std::nullopt;</highlight></codeline> |
| <codeline lineno="153"><highlight class="normal"><sp/><sp/><sp/><sp/>}</highlight></codeline> |
| <codeline lineno="154"><highlight class="normal"></highlight></codeline> |
| <codeline lineno="155"><highlight class="normal"><sp/><sp/><sp/><sp/>ICEBERG_ASSIGN_OR_RAISE(</highlight></codeline> |
| <codeline lineno="156"><highlight class="normal"><sp/><sp/><sp/><sp/><sp/><sp/><sp/><sp/>batch,<sp/>ProjectRecordBatch(std::move(batch),<sp/>context_->output_arrow_schema_,</highlight></codeline> |
| <codeline lineno="157"><highlight class="normal"><sp/><sp/><sp/><sp/><sp/><sp/><sp/><sp/><sp/><sp/><sp/><sp/><sp/><sp/><sp/><sp/><sp/><sp/><sp/><sp/><sp/><sp/><sp/><sp/><sp/><sp/><sp/><sp/><sp/><sp/><sp/><sp/><sp/><sp/>*read_schema_,<sp/>projection_,<sp/>metadata_context_,<sp/>pool_));</highlight></codeline> |
| <codeline lineno="158"><highlight class="normal"></highlight></codeline> |
| <codeline lineno="159"><highlight class="normal"><sp/><sp/><sp/><sp/>metadata_context_.next_file_pos<sp/>+=<sp/>batch->num_rows();</highlight></codeline> |
| <codeline lineno="160"><highlight class="normal"></highlight></codeline> |
| <codeline lineno="161"><highlight class="normal"><sp/><sp/><sp/><sp/><ref refid="structArrowArray" kindref="compound">ArrowArray</ref><sp/>arrow_array;</highlight></codeline> |
| <codeline lineno="162"><highlight class="normal"><sp/><sp/><sp/><sp/>ICEBERG_ARROW_RETURN_NOT_OK(::arrow::ExportRecordBatch(*batch,<sp/>&arrow_array));</highlight></codeline> |
| <codeline lineno="163"><highlight class="normal"><sp/><sp/><sp/><sp/></highlight><highlight class="keywordflow">return</highlight><highlight class="normal"><sp/>arrow_array;</highlight></codeline> |
| <codeline lineno="164"><highlight class="normal"><sp/><sp/>}</highlight></codeline> |
| <codeline lineno="165"><highlight class="normal"></highlight></codeline> |
| <codeline lineno="166"><highlight class="normal"><sp/><sp/></highlight><highlight class="comment">//<sp/>Close<sp/>the<sp/>reader<sp/>and<sp/>release<sp/>resources</highlight><highlight class="normal"></highlight></codeline> |
| <codeline lineno="167"><highlight class="normal"><sp/><sp/>Status<sp/>Close()<sp/>{</highlight></codeline> |
| <codeline lineno="168"><highlight class="normal"><sp/><sp/><sp/><sp/></highlight><highlight class="keywordflow">if</highlight><highlight class="normal"><sp/>(reader_<sp/>==<sp/></highlight><highlight class="keyword">nullptr</highlight><highlight class="normal">)<sp/>{</highlight></codeline> |
| <codeline lineno="169"><highlight class="normal"><sp/><sp/><sp/><sp/><sp/><sp/></highlight><highlight class="keywordflow">return</highlight><highlight class="normal"><sp/>{};<sp/><sp/></highlight><highlight class="comment">//<sp/>Already<sp/>closed</highlight><highlight class="normal"></highlight></codeline> |
| <codeline lineno="170"><highlight class="normal"><sp/><sp/><sp/><sp/>}</highlight></codeline> |
| <codeline lineno="171"><highlight class="normal"></highlight></codeline> |
| <codeline lineno="172"><highlight class="normal"><sp/><sp/><sp/><sp/></highlight><highlight class="keywordflow">if</highlight><highlight class="normal"><sp/>(context_<sp/>!=<sp/></highlight><highlight class="keyword">nullptr</highlight><highlight class="normal">)<sp/>{</highlight></codeline> |
| <codeline lineno="173"><highlight class="normal"><sp/><sp/><sp/><sp/><sp/><sp/>ICEBERG_ARROW_RETURN_NOT_OK(context_->record_batch_reader_->Close());</highlight></codeline> |
| <codeline lineno="174"><highlight class="normal"><sp/><sp/><sp/><sp/><sp/><sp/>context_.reset();</highlight></codeline> |
| <codeline lineno="175"><highlight class="normal"><sp/><sp/><sp/><sp/>}</highlight></codeline> |
| <codeline lineno="176"><highlight class="normal"></highlight></codeline> |
| <codeline lineno="177"><highlight class="normal"><sp/><sp/><sp/><sp/>reader_.reset();</highlight></codeline> |
| <codeline lineno="178"><highlight class="normal"><sp/><sp/><sp/><sp/>ICEBERG_ARROW_RETURN_NOT_OK(input_stream_->Close());</highlight></codeline> |
| <codeline lineno="179"><highlight class="normal"><sp/><sp/><sp/><sp/></highlight><highlight class="keywordflow">return</highlight><highlight class="normal"><sp/>{};</highlight></codeline> |
| <codeline lineno="180"><highlight class="normal"><sp/><sp/>}</highlight></codeline> |
| <codeline lineno="181"><highlight class="normal"></highlight></codeline> |
| <codeline lineno="182"><highlight class="normal"><sp/><sp/></highlight><highlight class="comment">//<sp/>Get<sp/>the<sp/>schema<sp/>of<sp/>the<sp/>data</highlight><highlight class="normal"></highlight></codeline> |
| <codeline lineno="183"><highlight class="normal"><sp/><sp/>Result<ArrowSchema><sp/><ref refid="classiceberg_1_1Schema" kindref="compound">Schema</ref>()<sp/>{</highlight></codeline> |
| <codeline lineno="184"><highlight class="normal"><sp/><sp/><sp/><sp/></highlight><highlight class="keywordflow">if</highlight><highlight class="normal"><sp/>(!context_)<sp/>{</highlight></codeline> |
| <codeline lineno="185"><highlight class="normal"><sp/><sp/><sp/><sp/><sp/><sp/>ICEBERG_RETURN_UNEXPECTED(InitReadContext());</highlight></codeline> |
| <codeline lineno="186"><highlight class="normal"><sp/><sp/><sp/><sp/>}</highlight></codeline> |
| <codeline lineno="187"><highlight class="normal"></highlight></codeline> |
| <codeline lineno="188"><highlight class="normal"><sp/><sp/><sp/><sp/><ref refid="structArrowSchema" kindref="compound">ArrowSchema</ref><sp/>arrow_schema;</highlight></codeline> |
| <codeline lineno="189"><highlight class="normal"><sp/><sp/><sp/><sp/>ICEBERG_ARROW_RETURN_NOT_OK(</highlight></codeline> |
| <codeline lineno="190"><highlight class="normal"><sp/><sp/><sp/><sp/><sp/><sp/><sp/><sp/>::arrow::ExportSchema(*context_->output_arrow_schema_,<sp/>&arrow_schema));</highlight></codeline> |
| <codeline lineno="191"><highlight class="normal"><sp/><sp/><sp/><sp/></highlight><highlight class="keywordflow">return</highlight><highlight class="normal"><sp/>arrow_schema;</highlight></codeline> |
| <codeline lineno="192"><highlight class="normal"><sp/><sp/>}</highlight></codeline> |
| <codeline lineno="193"><highlight class="normal"></highlight></codeline> |
| <codeline lineno="194"><highlight class="normal"><sp/><sp/>Result<std::unordered_map<std::string,<sp/>std::string>><sp/>Metadata()<sp/>{</highlight></codeline> |
| <codeline lineno="195"><highlight class="normal"><sp/><sp/><sp/><sp/></highlight><highlight class="keywordflow">if</highlight><highlight class="normal"><sp/>(reader_<sp/>==<sp/></highlight><highlight class="keyword">nullptr</highlight><highlight class="normal">)<sp/>{</highlight></codeline> |
| <codeline lineno="196"><highlight class="normal"><sp/><sp/><sp/><sp/><sp/><sp/></highlight><highlight class="keywordflow">return</highlight><highlight class="normal"><sp/>Invalid(</highlight><highlight class="stringliteral">"Reader<sp/>is<sp/>not<sp/>opened"</highlight><highlight class="normal">);</highlight></codeline> |
| <codeline lineno="197"><highlight class="normal"><sp/><sp/><sp/><sp/>}</highlight></codeline> |
| <codeline lineno="198"><highlight class="normal"></highlight></codeline> |
| <codeline lineno="199"><highlight class="normal"><sp/><sp/><sp/><sp/></highlight><highlight class="keyword">auto</highlight><highlight class="normal"><sp/>metadata<sp/>=<sp/>reader_->parquet_reader()->metadata();</highlight></codeline> |
| <codeline lineno="200"><highlight class="normal"><sp/><sp/><sp/><sp/></highlight><highlight class="keywordflow">if</highlight><highlight class="normal"><sp/>(!metadata)<sp/>{</highlight></codeline> |
| <codeline lineno="201"><highlight class="normal"><sp/><sp/><sp/><sp/><sp/><sp/></highlight><highlight class="keywordflow">return</highlight><highlight class="normal"><sp/>Invalid(</highlight><highlight class="stringliteral">"Failed<sp/>to<sp/>get<sp/>Parquet<sp/>file<sp/>metadata"</highlight><highlight class="normal">);</highlight></codeline> |
| <codeline lineno="202"><highlight class="normal"><sp/><sp/><sp/><sp/>}</highlight></codeline> |
| <codeline lineno="203"><highlight class="normal"></highlight></codeline> |
| <codeline lineno="204"><highlight class="normal"><sp/><sp/><sp/><sp/></highlight><highlight class="keyword">const</highlight><highlight class="normal"><sp/></highlight><highlight class="keyword">auto</highlight><highlight class="normal">&<sp/>kv_metadata<sp/>=<sp/>metadata->key_value_metadata();</highlight></codeline> |
| <codeline lineno="205"><highlight class="normal"><sp/><sp/><sp/><sp/></highlight><highlight class="keywordflow">if</highlight><highlight class="normal"><sp/>(!kv_metadata)<sp/>{</highlight></codeline> |
| <codeline lineno="206"><highlight class="normal"><sp/><sp/><sp/><sp/><sp/><sp/></highlight><highlight class="keywordflow">return</highlight><highlight class="normal"><sp/>std::unordered_map<std::string,<sp/>std::string>{};</highlight></codeline> |
| <codeline lineno="207"><highlight class="normal"><sp/><sp/><sp/><sp/>}</highlight></codeline> |
| <codeline lineno="208"><highlight class="normal"></highlight></codeline> |
| <codeline lineno="209"><highlight class="normal"><sp/><sp/><sp/><sp/>std::unordered_map<std::string,<sp/>std::string><sp/>metadata_map;</highlight></codeline> |
| <codeline lineno="210"><highlight class="normal"><sp/><sp/><sp/><sp/>kv_metadata->ToUnorderedMap(&metadata_map);</highlight></codeline> |
| <codeline lineno="211"><highlight class="normal"></highlight></codeline> |
| <codeline lineno="212"><highlight class="normal"><sp/><sp/><sp/><sp/></highlight><highlight class="keywordflow">return</highlight><highlight class="normal"><sp/>metadata_map;</highlight></codeline> |
| <codeline lineno="213"><highlight class="normal"><sp/><sp/>}</highlight></codeline> |
| <codeline lineno="214"><highlight class="normal"></highlight></codeline> |
| <codeline lineno="215"><highlight class="normal"><sp/></highlight><highlight class="keyword">private</highlight><highlight class="normal">:</highlight></codeline> |
| <codeline lineno="216"><highlight class="normal"><sp/><sp/>Status<sp/>InitReadContext()<sp/>{</highlight></codeline> |
| <codeline lineno="217"><highlight class="normal"><sp/><sp/><sp/><sp/>context_<sp/>=<sp/>std::make_unique<ReadContext>();</highlight></codeline> |
| <codeline lineno="218"><highlight class="normal"></highlight></codeline> |
| <codeline lineno="219"><highlight class="normal"><sp/><sp/><sp/><sp/></highlight><highlight class="comment">//<sp/>Build<sp/>the<sp/>output<sp/>Arrow<sp/>schema</highlight><highlight class="normal"></highlight></codeline> |
| <codeline lineno="220"><highlight class="normal"><sp/><sp/><sp/><sp/><ref refid="structArrowSchema" kindref="compound">ArrowSchema</ref><sp/>arrow_schema;</highlight></codeline> |
| <codeline lineno="221"><highlight class="normal"><sp/><sp/><sp/><sp/>ICEBERG_RETURN_UNEXPECTED(ToArrowSchema(*read_schema_,<sp/>&arrow_schema));</highlight></codeline> |
| <codeline lineno="222"><highlight class="normal"><sp/><sp/><sp/><sp/>ICEBERG_ARROW_ASSIGN_OR_RETURN(context_->output_arrow_schema_,</highlight></codeline> |
| <codeline lineno="223"><highlight class="normal"><sp/><sp/><sp/><sp/><sp/><sp/><sp/><sp/><sp/><sp/><sp/><sp/><sp/><sp/><sp/><sp/><sp/><sp/><sp/><sp/><sp/><sp/><sp/><sp/><sp/><sp/><sp/><sp/><sp/><sp/><sp/><sp/><sp/><sp/><sp/>::arrow::ImportSchema(&arrow_schema));</highlight></codeline> |
| <codeline lineno="224"><highlight class="normal"></highlight></codeline> |
| <codeline lineno="225"><highlight class="normal"><sp/><sp/><sp/><sp/></highlight><highlight class="comment">//<sp/>Row<sp/>group<sp/>pruning<sp/>based<sp/>on<sp/>the<sp/>split</highlight><highlight class="normal"></highlight></codeline> |
| <codeline lineno="226"><highlight class="normal"><sp/><sp/><sp/><sp/></highlight><highlight class="comment">//<sp/>TODO(gangwu):<sp/>add<sp/>row<sp/>group<sp/>filtering<sp/>based<sp/>on<sp/>zone<sp/>map,<sp/>bloom<sp/>filter,<sp/>etc.</highlight><highlight class="normal"></highlight></codeline> |
| <codeline lineno="227"><highlight class="normal"><sp/><sp/><sp/><sp/>std::vector<int><sp/>row_group_indices;</highlight></codeline> |
| <codeline lineno="228"><highlight class="normal"><sp/><sp/><sp/><sp/></highlight><highlight class="keywordflow">if</highlight><highlight class="normal"><sp/>(split_.has_value())<sp/>{</highlight></codeline> |
| <codeline lineno="229"><highlight class="normal"><sp/><sp/><sp/><sp/><sp/><sp/></highlight><highlight class="keyword">auto</highlight><highlight class="normal"><sp/>metadata<sp/>=<sp/>reader_->parquet_reader()->metadata();</highlight></codeline> |
| <codeline lineno="230"><highlight class="normal"><sp/><sp/><sp/><sp/><sp/><sp/></highlight><highlight class="keywordflow">for</highlight><highlight class="normal"><sp/>(</highlight><highlight class="keywordtype">int</highlight><highlight class="normal"><sp/>i<sp/>=<sp/>0;<sp/>i<sp/><<sp/>metadata->num_row_groups();<sp/>++i)<sp/>{</highlight></codeline> |
| <codeline lineno="231"><highlight class="normal"><sp/><sp/><sp/><sp/><sp/><sp/><sp/><sp/></highlight><highlight class="keyword">auto</highlight><highlight class="normal"><sp/>row_group_offset<sp/>=<sp/>metadata->RowGroup(i)->file_offset();</highlight></codeline> |
| <codeline lineno="232"><highlight class="normal"><sp/><sp/><sp/><sp/><sp/><sp/><sp/><sp/></highlight><highlight class="keywordflow">if</highlight><highlight class="normal"><sp/>(row_group_offset<sp/>>=<sp/>split_->offset<sp/>&&</highlight></codeline> |
| <codeline lineno="233"><highlight class="normal"><sp/><sp/><sp/><sp/><sp/><sp/><sp/><sp/><sp/><sp/><sp/><sp/>row_group_offset<sp/><<sp/>split_->offset<sp/>+<sp/>split_->length)<sp/>{</highlight></codeline> |
| <codeline lineno="234"><highlight class="normal"><sp/><sp/><sp/><sp/><sp/><sp/><sp/><sp/><sp/><sp/>row_group_indices.push_back(i);</highlight></codeline> |
| <codeline lineno="235"><highlight class="normal"><sp/><sp/><sp/><sp/><sp/><sp/><sp/><sp/>}<sp/></highlight><highlight class="keywordflow">else</highlight><highlight class="normal"><sp/></highlight><highlight class="keywordflow">if</highlight><highlight class="normal"><sp/>(row_group_offset<sp/>>=<sp/>split_->offset<sp/>+<sp/>split_->length)<sp/>{</highlight></codeline> |
| <codeline lineno="236"><highlight class="normal"><sp/><sp/><sp/><sp/><sp/><sp/><sp/><sp/><sp/><sp/></highlight><highlight class="keywordflow">break</highlight><highlight class="normal">;</highlight></codeline> |
| <codeline lineno="237"><highlight class="normal"><sp/><sp/><sp/><sp/><sp/><sp/><sp/><sp/>}<sp/></highlight><highlight class="keywordflow">else</highlight><highlight class="normal"><sp/>{</highlight></codeline> |
| <codeline lineno="238"><highlight class="normal"><sp/><sp/><sp/><sp/><sp/><sp/><sp/><sp/><sp/><sp/>metadata_context_.next_file_pos<sp/>+=<sp/>metadata->RowGroup(i)->num_rows();</highlight></codeline> |
| <codeline lineno="239"><highlight class="normal"><sp/><sp/><sp/><sp/><sp/><sp/><sp/><sp/>}</highlight></codeline> |
| <codeline lineno="240"><highlight class="normal"><sp/><sp/><sp/><sp/><sp/><sp/>}</highlight></codeline> |
| <codeline lineno="241"><highlight class="normal"><sp/><sp/><sp/><sp/>}<sp/></highlight><highlight class="keywordflow">else</highlight><highlight class="normal"><sp/>{</highlight></codeline> |
| <codeline lineno="242"><highlight class="normal"><sp/><sp/><sp/><sp/><sp/><sp/>row_group_indices.resize(reader_->parquet_reader()->metadata()->num_row_groups());</highlight></codeline> |
| <codeline lineno="243"><highlight class="normal"><sp/><sp/><sp/><sp/><sp/><sp/>std::iota(row_group_indices.begin(),<sp/>row_group_indices.end(),<sp/>0);<sp/><sp/></highlight><highlight class="comment">//<sp/>NOLINT</highlight><highlight class="normal"></highlight></codeline> |
| <codeline lineno="244"><highlight class="normal"><sp/><sp/><sp/><sp/>}</highlight></codeline> |
| <codeline lineno="245"><highlight class="normal"></highlight></codeline> |
| <codeline lineno="246"><highlight class="normal"><sp/><sp/><sp/><sp/></highlight><highlight class="comment">//<sp/>Create<sp/>the<sp/>record<sp/>batch<sp/>reader</highlight><highlight class="normal"></highlight></codeline> |
| <codeline lineno="247"><highlight class="normal"><sp/><sp/><sp/><sp/></highlight><highlight class="keywordflow">if</highlight><highlight class="normal"><sp/>(row_group_indices.empty())<sp/>{</highlight></codeline> |
| <codeline lineno="248"><highlight class="normal"><sp/><sp/><sp/><sp/><sp/><sp/></highlight><highlight class="comment">//<sp/>None<sp/>of<sp/>the<sp/>row<sp/>groups<sp/>are<sp/>selected,<sp/>return<sp/>an<sp/>empty<sp/>record<sp/>batch<sp/>reader</highlight><highlight class="normal"></highlight></codeline> |
| <codeline lineno="249"><highlight class="normal"><sp/><sp/><sp/><sp/><sp/><sp/>context_->record_batch_reader_<sp/>=<sp/>std::make_unique<EmptyRecordBatchReader>();</highlight></codeline> |
| <codeline lineno="250"><highlight class="normal"><sp/><sp/><sp/><sp/>}<sp/></highlight><highlight class="keywordflow">else</highlight><highlight class="normal"><sp/>{</highlight></codeline> |
| <codeline lineno="251"><highlight class="normal"><sp/><sp/><sp/><sp/><sp/><sp/></highlight><highlight class="keyword">auto</highlight><highlight class="normal"><sp/>column_indices<sp/>=<sp/>SelectedColumnIndices(projection_);</highlight></codeline> |
| <codeline lineno="252"><highlight class="normal"><sp/><sp/><sp/><sp/><sp/><sp/>ICEBERG_ARROW_ASSIGN_OR_RETURN(</highlight></codeline> |
| <codeline lineno="253"><highlight class="normal"><sp/><sp/><sp/><sp/><sp/><sp/><sp/><sp/><sp/><sp/>context_->record_batch_reader_,</highlight></codeline> |
| <codeline lineno="254"><highlight class="normal"><sp/><sp/><sp/><sp/><sp/><sp/><sp/><sp/><sp/><sp/>reader_->GetRecordBatchReader(row_group_indices,<sp/>column_indices));</highlight></codeline> |
| <codeline lineno="255"><highlight class="normal"><sp/><sp/><sp/><sp/>}</highlight></codeline> |
| <codeline lineno="256"><highlight class="normal"></highlight></codeline> |
| <codeline lineno="257"><highlight class="normal"><sp/><sp/><sp/><sp/></highlight><highlight class="keywordflow">return</highlight><highlight class="normal"><sp/>{};</highlight></codeline> |
| <codeline lineno="258"><highlight class="normal"><sp/><sp/>}</highlight></codeline> |
| <codeline lineno="259"><highlight class="normal"></highlight></codeline> |
| <codeline lineno="260"><highlight class="normal"><sp/></highlight><highlight class="keyword">private</highlight><highlight class="normal">:</highlight></codeline> |
| <codeline lineno="261"><highlight class="normal"><sp/><sp/></highlight><highlight class="comment">//<sp/>TODO(gangwu):<sp/>make<sp/>memory<sp/>pool<sp/>configurable</highlight><highlight class="normal"></highlight></codeline> |
| <codeline lineno="262"><highlight class="normal"><sp/><sp/>::arrow::MemoryPool*<sp/>pool_<sp/>=<sp/>::arrow::default_memory_pool();</highlight></codeline> |
| <codeline lineno="263"><highlight class="normal"><sp/><sp/></highlight><highlight class="comment">//<sp/>The<sp/>split<sp/>to<sp/>read<sp/>from<sp/>the<sp/>Parquet<sp/>file.</highlight><highlight class="normal"></highlight></codeline> |
| <codeline lineno="264"><highlight class="normal"><sp/><sp/>std::optional<Split><sp/>split_;</highlight></codeline> |
| <codeline lineno="265"><highlight class="normal"><sp/><sp/></highlight><highlight class="comment">//<sp/>Schema<sp/>to<sp/>read<sp/>from<sp/>the<sp/>Parquet<sp/>file.</highlight><highlight class="normal"></highlight></codeline> |
| <codeline lineno="266"><highlight class="normal"><sp/><sp/>std::shared_ptr<::iceberg::Schema><sp/>read_schema_;</highlight></codeline> |
| <codeline lineno="267"><highlight class="normal"><sp/><sp/></highlight><highlight class="comment">//<sp/>The<sp/>projection<sp/>result<sp/>to<sp/>apply<sp/>to<sp/>the<sp/>read<sp/>schema.</highlight><highlight class="normal"></highlight></codeline> |
| <codeline lineno="268"><highlight class="normal"><sp/><sp/><ref refid="structiceberg_1_1SchemaProjection" kindref="compound">SchemaProjection</ref><sp/>projection_;</highlight></codeline> |
| <codeline lineno="269"><highlight class="normal"><sp/><sp/></highlight><highlight class="comment">//<sp/>The<sp/>input<sp/>stream<sp/>to<sp/>read<sp/>Parquet<sp/>file.</highlight><highlight class="normal"></highlight></codeline> |
| <codeline lineno="270"><highlight class="normal"><sp/><sp/>std::shared_ptr<::arrow::io::RandomAccessFile><sp/>input_stream_;</highlight></codeline> |
| <codeline lineno="271"><highlight class="normal"><sp/><sp/></highlight><highlight class="comment">//<sp/>Parquet<sp/>file<sp/>reader<sp/>to<sp/>create<sp/>RecordBatchReader.</highlight><highlight class="normal"></highlight></codeline> |
| <codeline lineno="272"><highlight class="normal"><sp/><sp/>std::unique_ptr<::parquet::arrow::FileReader><sp/>reader_;</highlight></codeline> |
| <codeline lineno="273"><highlight class="normal"><sp/><sp/></highlight><highlight class="comment">//<sp/>Metadata<sp/>column<sp/>context<sp/>for<sp/>populating<sp/>_file<sp/>and<sp/>_pos<sp/>columns.</highlight><highlight class="normal"></highlight></codeline> |
| <codeline lineno="274"><highlight class="normal"><sp/><sp/><ref refid="structiceberg_1_1arrow_1_1MetadataColumnContext" kindref="compound">arrow::MetadataColumnContext</ref><sp/>metadata_context_;</highlight></codeline> |
| <codeline lineno="275"><highlight class="normal"><sp/><sp/></highlight><highlight class="comment">//<sp/>The<sp/>context<sp/>to<sp/>keep<sp/>track<sp/>of<sp/>the<sp/>reading<sp/>progress.</highlight><highlight class="normal"></highlight></codeline> |
| <codeline lineno="276"><highlight class="normal"><sp/><sp/>std::unique_ptr<ReadContext><sp/>context_;</highlight></codeline> |
| <codeline lineno="277"><highlight class="normal">};</highlight></codeline> |
| <codeline lineno="278"><highlight class="normal"></highlight></codeline> |
| <codeline lineno="279"><highlight class="normal">ParquetReader::~ParquetReader()<sp/>=<sp/></highlight><highlight class="keywordflow">default</highlight><highlight class="normal">;</highlight></codeline> |
| <codeline lineno="280"><highlight class="normal"></highlight></codeline> |
| <codeline lineno="281"><highlight class="normal">Result<std::optional<ArrowArray>><sp/><ref refid="classiceberg_1_1parquet_1_1ParquetReader_1a34c1a71ce9d5eaf14c1854ca85575bac" kindref="member">ParquetReader::Next</ref>()<sp/>{<sp/></highlight><highlight class="keywordflow">return</highlight><highlight class="normal"><sp/>impl_->Next();<sp/>}</highlight></codeline> |
| <codeline lineno="282"><highlight class="normal"></highlight></codeline> |
| <codeline lineno="283"><highlight class="normal">Result<ArrowSchema><sp/><ref refid="classiceberg_1_1parquet_1_1ParquetReader_1a24fd6d5fcf479c6f6b51db925ad1b25e" kindref="member">ParquetReader::Schema</ref>()<sp/>{<sp/></highlight><highlight class="keywordflow">return</highlight><highlight class="normal"><sp/>impl_->Schema();<sp/>}</highlight></codeline> |
| <codeline lineno="284"><highlight class="normal"></highlight></codeline> |
| <codeline lineno="285"><highlight class="normal">Result<std::unordered_map<std::string,<sp/>std::string>><sp/><ref refid="classiceberg_1_1parquet_1_1ParquetReader_1ab372da8bb16449db4ff59a319981947a" kindref="member">ParquetReader::Metadata</ref>()<sp/>{</highlight></codeline> |
| <codeline lineno="286"><highlight class="normal"><sp/><sp/></highlight><highlight class="keywordflow">return</highlight><highlight class="normal"><sp/>impl_->Metadata();</highlight></codeline> |
| <codeline lineno="287"><highlight class="normal">}</highlight></codeline> |
| <codeline lineno="288"><highlight class="normal"></highlight></codeline> |
| <codeline lineno="289"><highlight class="normal">Status<sp/><ref refid="classiceberg_1_1parquet_1_1ParquetReader_1ab75a19e9370963e6790f4f523e5ced32" kindref="member">ParquetReader::Open</ref>(</highlight><highlight class="keyword">const</highlight><highlight class="normal"><sp/>ReaderOptions&<sp/>options)<sp/>{</highlight></codeline> |
| <codeline lineno="290"><highlight class="normal"><sp/><sp/>impl_<sp/>=<sp/>std::make_unique<Impl>();</highlight></codeline> |
| <codeline lineno="291"><highlight class="normal"><sp/><sp/></highlight><highlight class="keywordflow">return</highlight><highlight class="normal"><sp/>impl_->Open(options);</highlight></codeline> |
| <codeline lineno="292"><highlight class="normal">}</highlight></codeline> |
| <codeline lineno="293"><highlight class="normal"></highlight></codeline> |
| <codeline lineno="294"><highlight class="normal">Status<sp/><ref refid="classiceberg_1_1parquet_1_1ParquetReader_1a4aecbbf7865ac94054fe5c9f559c6aea" kindref="member">ParquetReader::Close</ref>()<sp/>{<sp/></highlight><highlight class="keywordflow">return</highlight><highlight class="normal"><sp/>impl_->Close();<sp/>}</highlight></codeline> |
| <codeline lineno="295"><highlight class="normal"></highlight></codeline> |
| <codeline lineno="296"><highlight class="normal"></highlight><highlight class="keywordtype">void</highlight><highlight class="normal"><sp/>RegisterReader()<sp/>{</highlight></codeline> |
| <codeline lineno="297"><highlight class="normal"><sp/><sp/></highlight><highlight class="keyword">static</highlight><highlight class="normal"><sp/>ReaderFactoryRegistry<sp/>parquet_reader_register(</highlight></codeline> |
| <codeline lineno="298"><highlight class="normal"><sp/><sp/><sp/><sp/><sp/><sp/>FileFormatType::kParquet,<sp/>[]()<sp/>-><sp/>Result<std::unique_ptr<Reader>><sp/>{</highlight></codeline> |
| <codeline lineno="299"><highlight class="normal"><sp/><sp/><sp/><sp/><sp/><sp/><sp/><sp/></highlight><highlight class="keywordflow">return</highlight><highlight class="normal"><sp/>std::make_unique<ParquetReader>();</highlight></codeline> |
| <codeline lineno="300"><highlight class="normal"><sp/><sp/><sp/><sp/><sp/><sp/>});</highlight></codeline> |
| <codeline lineno="301"><highlight class="normal">}</highlight></codeline> |
| <codeline lineno="302"><highlight class="normal"></highlight></codeline> |
| <codeline lineno="303"><highlight class="normal">}<sp/><sp/></highlight><highlight class="comment">//<sp/>namespace<sp/>iceberg::parquet</highlight><highlight class="normal"></highlight></codeline> |
| </programlisting> |
| <location file="iceberg/parquet/parquet_reader.cc"/> |
| </compounddef> |
| </doxygen> |
