PARQUET-2467: Archive this repo (#504)
* Update the README
* Remove archived contents
diff --git a/.clang-format b/.clang-format
deleted file mode 100644
index 06453df..0000000
--- a/.clang-format
+++ /dev/null
@@ -1,20 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
----
-BasedOnStyle: Google
-DerivePointerAlignment: false
-ColumnLimit: 90
diff --git a/.clang-tidy b/.clang-tidy
deleted file mode 100644
index bae34bc..0000000
--- a/.clang-tidy
+++ /dev/null
@@ -1,30 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
----
-Checks: 'clang-diagnostic-*,clang-analyzer-*,-clang-analyzer-alpha*,google-.*,modernize-.*,readablity-.*'
-HeaderFilterRegex: 'parquet/.*'
-AnalyzeTemporaryDtors: true
-CheckOptions:
- - key: google-readability-braces-around-statements.ShortStatementLines
- value: '1'
- - key: google-readability-function-size.StatementThreshold
- value: '800'
- - key: google-readability-namespace-comments.ShortNamespaceLines
- value: '10'
- - key: google-readability-namespace-comments.SpacesBeforeComments
- value: '2'
diff --git a/.clang-tidy-ignore b/.clang-tidy-ignore
deleted file mode 100644
index 8b13789..0000000
--- a/.clang-tidy-ignore
+++ /dev/null
@@ -1 +0,0 @@
-
diff --git a/.parquetcppversion b/.parquetcppversion
deleted file mode 100644
index f825f7c..0000000
--- a/.parquetcppversion
+++ /dev/null
@@ -1 +0,0 @@
-1.5.1-SNAPSHOT
diff --git a/.travis.yml b/.travis.yml
deleted file mode 100644
index e1faf68..0000000
--- a/.travis.yml
+++ /dev/null
@@ -1,108 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-sudo: required
-dist: trusty
-
-language: cpp
-
-cache:
- ccache: true
-
-addons:
- apt:
- sources:
- - ubuntu-toolchain-r-test
- packages:
- - gcc-4.9
- - g++-4.9
- - valgrind
- - libboost-dev
- - libboost-filesystem-dev
- - libboost-regex-dev
- - libboost-system-dev
- - libboost-program-options-dev
- - libboost-test-dev
- - libssl-dev
- - libtool
- - bison
- - flex
- - pkg-config
-
-matrix:
- fast_finish: true
- include:
- - compiler: gcc
- os: linux
- before_script:
- - export PARQUET_CXXFLAGS="-DARROW_NO_DEPRECATED_API"
- - export PARQUET_TRAVIS_VALGRIND=1
- - source $TRAVIS_BUILD_DIR/ci/before_script_travis.sh
- - compiler: clang
- os: linux
- before_script:
- - export PARQUET_CXXFLAGS="-DARROW_NO_DEPRECATED_API"
- - source $TRAVIS_BUILD_DIR/ci/before_script_travis.sh
- - compiler: clang
- os: osx
- osx_image: xcode6.4
- addons:
- before_script:
- - export PARQUET_CXXFLAGS="-DARROW_NO_DEPRECATED_API"
- - source $TRAVIS_BUILD_DIR/ci/before_script_travis.sh
- before_install:
- - mkdir $TRAVIS_BUILD_DIR/parquet-build
- - pushd $TRAVIS_BUILD_DIR/parquet-build
- - compiler: gcc
- os: linux
- env: PARQUET_BUILD_GROUP=toolchain
- before_script:
- script:
- - export PARQUET_CXXFLAGS="-DARROW_NO_DEPRECATED_API"
- - $TRAVIS_BUILD_DIR/ci/travis_script_static.sh
- - compiler: gcc
- os: linux
- env: PARQUET_BUILD_GROUP=toolchain
- before_script:
- script:
- - $TRAVIS_BUILD_DIR/ci/travis_script_toolchain.sh
-
-# PARQUET-626: revisit llvm toolchain when/if llvm.org apt repo resurfaces
-
-# before_install:
-# - wget -O - http://llvm.org/apt/llvm-snapshot.gpg.key|sudo apt-key add -
-# - sudo apt-get install -y software-properties-common
-# - sudo add-apt-repository -y ppa:george-edison55/cmake-3.x
-# - sudo apt-add-repository -y "deb http://llvm.org/apt/trusty/ llvm-toolchain-trusty-3.7
-# main"
-# - sudo apt-add-repository -y "deb http://llvm.org/apt/trusty/ llvm-toolchain-trusty-3.8
-# main"
-# - sudo apt-get update
-# - sudo apt-get install -y clang-tidy-3.7 clang-format-3.7 cmake
-# - mkdir $TRAVIS_BUILD_DIR/parquet-build
-# - pushd $TRAVIS_BUILD_DIR/parquet-build
-
-before_install:
-- mkdir $TRAVIS_BUILD_DIR/parquet-build
-- pushd $TRAVIS_BUILD_DIR/parquet-build
-
-script:
-- $TRAVIS_BUILD_DIR/ci/travis_script_cpp.sh
-
-env:
- global:
- - secure: "sGGfe2N9i2dItR0FryQYZDdzdi9gl6QFXxGWxxXWfPz+mideJWaMLSA17dsWkbrag5GlAcs+8BRGWWagAXicPlOqD1tKKJqZxA4PEPDECny26Sh/pebZvmvnAV5136sXkKXMQkcTfoo3v22zYdVthVa+8EQTXKxGzhbL61AosSk="
diff --git a/CHANGELOG b/CHANGELOG
deleted file mode 100644
index 4673741..0000000
--- a/CHANGELOG
+++ /dev/null
@@ -1,503 +0,0 @@
-Parquet C++ 1.5.0
---------------------------------------------------------------------------------
-## Bug
- * [PARQUET-979] - [C++] Limit size of min, max or disable stats for long binary types
- * [PARQUET-1071] - [C++] parquet::arrow::FileWriter::Close is not idempotent
- * [PARQUET-1349] - [C++] PARQUET_RPATH_ORIGIN is not picked by the build
- * [PARQUET-1334] - [C++] memory_map parameter seems missleading in parquet file opener
- * [PARQUET-1333] - [C++] Reading of files with dictionary size 0 fails on Windows with bad_alloc
- * [PARQUET-1283] - [C++] FormatStatValue appends trailing space to string and int96
- * [PARQUET-1270] - [C++] Executable tools do not get installed
- * [PARQUET-1272] - [C++] ScanFileContents reports wrong row count for nested columns
- * [PARQUET-1268] - [C++] Conversion of Arrow null list columns fails
- * [PARQUET-1255] - [C++] Exceptions thrown in some tests
- * [PARQUET-1358] - [C++] index_page_offset should be unset as it is not supported.
- * [PARQUET-1357] - [C++] FormatStatValue truncates binary statistics on zero character
- * [PARQUET-1319] - [C++] Pass BISON_EXECUTABLE to Thrift EP for MacOS
- * [PARQUET-1313] - [C++] Compilation failure with VS2017
- * [PARQUET-1315] - [C++] ColumnChunkMetaData.has_dictionary_page() should return bool, not int64_t
- * [PARQUET-1307] - [C++] memory-test fails with latest Arrow
- * [PARQUET-1274] - [Python] SegFault in pyarrow.parquet.write_table with specific options
- * [PARQUET-1209] - locally defined symbol ... imported in function ..
- * [PARQUET-1245] - [C++] Segfault when writing Arrow table with duplicate columns
- * [PARQUET-1273] - [Python] Error writing to partitioned Parquet dataset
- * [PARQUET-1384] - [C++] Clang compiler warnings in bloom_filter-test.cc
-
-## Improvement
- * [PARQUET-1348] - [C++] Allow Arrow FileWriter To Write FileMetaData
- * [PARQUET-1346] - [C++] Protect against null values data in empty Arrow array
- * [PARQUET-1340] - [C++] Fix Travis Ci valgrind errors related to std::random_device
- * [PARQUET-1323] - [C++] Fix compiler warnings with clang-6.0
- * [PARQUET-1279] - Use ASSERT_NO_FATAIL_FAILURE in C++ unit tests
- * [PARQUET-1262] - [C++] Use the same BOOST_ROOT and Boost_NAMESPACE for Thrift
- * [PARQUET-1267] - replace "unsafe" std::equal by std::memcmp
- * [PARQUET-1360] - [C++] Minor API + style changes follow up to PARQUET-1348
- * [PARQUET-1166] - [API Proposal] Add GetRecordBatchReader in parquet/arrow/reader.h
- * [PARQUET-1378] - [c++] Allow RowGroups with zero rows to be written
- * [PARQUET-1256] - [C++] Add --print-key-value-metadata option to parquet_reader tool
- * [PARQUET-1276] - [C++] Reduce the amount of memory used for writing null decimal values
-
-## New Feature
- * [PARQUET-1392] - [C++] Supply row group indices to parquet::arrow::FileReader::ReadTable
-
-## Sub-task
- * [PARQUET-1227] - Thrift crypto metadata structures
- * [PARQUET-1332] - [C++] Add bloom filter utility class
-
-## Task
- * [PARQUET-1350] - [C++] Use abstract ResizableBuffer instead of concrete PoolBuffer
- * [PARQUET-1366] - [C++] Streamline use of Arrow bit-util.h
- * [PARQUET-1308] - [C++] parquet::arrow should use thread pool, not ParallelFor
- * [PARQUET-1382] - [C++] Prepare for arrow::test namespace removal
- * [PARQUET-1372] - [C++] Add an API to allow writing RowGroups based on their size rather than num_rows
-
-
-Parquet C++ 1.4.0
---------------------------------------------------------------------------------
-## Bug
- * [PARQUET-1193] - [CPP] Implement ColumnOrder to support min_value and max_value
- * [PARQUET-1180] - C++: Fix behaviour of num_children element of primitive nodes
- * [PARQUET-1146] - C++: Add macOS-compatible sha512sum call to release verify script
- * [PARQUET-1167] - [C++] FieldToNode function should return a status when throwing an exception
- * [PARQUET-1175] - [C++] Fix usage of deprecated Arrow API
- * [PARQUET-1113] - [C++] Incorporate fix from ARROW-1601 on bitmap read path
- * [PARQUET-1111] - dev/release/verify-release-candidate has stale help
- * [PARQUET-1109] - C++: Update release verification script to SHA512
- * [PARQUET-1179] - [C++] Support Apache Thrift 0.11
- * [PARQUET-1226] - [C++] Fix new build warnings with clang 5.0
- * [PARQUET-1233] - [CPP ]Enable option to switch between stl classes and boost classes for thrift header
- * [PARQUET-1205] - Fix msvc static build
- * [PARQUET-1210] - [C++] Boost 1.66 compilation fails on Windows on linkage stage
-
-## Improvement
- * [PARQUET-1092] - [C++] Write Arrow tables with chunked columns
- * [PARQUET-1086] - [C++] Remove usage of arrow/util/compiler-util.h after 1.3.0 release
- * [PARQUET-1097] - [C++] Account for Arrow API deprecation in ARROW-1511
- * [PARQUET-1150] - C++: Hide statically linked boost symbols
- * [PARQUET-1151] - [C++] Add build options / configuration to use static runtime libraries with MSVC
- * [PARQUET-1147] - [C++] Account for API deprecation / change in ARROW-1671
- * [PARQUET-1162] - C++: Update dev/README after migration to Gitbox
- * [PARQUET-1165] - [C++] Pin clang-format version to 4.0
- * [PARQUET-1164] - [C++] Follow API changes in ARROW-1808
- * [PARQUET-1177] - [C++] Add more extensive compiler warnings when using Clang
- * [PARQUET-1110] - [C++] Release verification script for Windows
- * [PARQUET-859] - [C++] Flatten parquet/file directory
- * [PARQUET-1220] - [C++] Don't build Thrift examples and tutorials in the ExternalProject
- * [PARQUET-1219] - [C++] Update release-candidate script links to gitbox
- * [PARQUET-1196] - [C++] Provide a parquet_arrow example project incl. CMake setup
- * [PARQUET-1200] - [C++] Support reading a single Arrow column from a Parquet file
-
-## New Feature
- * [PARQUET-1095] - [C++] Read and write Arrow decimal values
- * [PARQUET-970] - Add Add Lz4 and Zstd compression codecs
-
-## Task
- * [PARQUET-1221] - [C++] Extend release README
- * [PARQUET-1225] - NaN values may lead to incorrect filtering under certain circumstances
-
-
-Parquet C++ 1.3.1
---------------------------------------------------------------------------------
-## Bug
- * [PARQUET-1105] - [CPP] Remove libboost_system dependency
- * [PARQUET-1138] - [C++] Fix compilation with Arrow 0.7.1
- * [PARQUET-1123] - [C++] Update parquet-cpp to use Arrow's AssertArraysEqual
- * [PARQUET-1121] - C++: DictionaryArrays of NullType cannot be written
- * [PARQUET-1139] - Add license to cmake_modules/parquet-cppConfig.cmake.in
-
-## Improvement
- * [PARQUET-1140] - [C++] Fail on RAT errors in CI
- * [PARQUET-1070] - Add CPack support to the build
-
-
-Parquet C++ 1.3.0
---------------------------------------------------------------------------------
-## Bug
- * [PARQUET-1098] - [C++] Install new header in parquet/util
- * [PARQUET-1085] - [C++] Backwards compatibility from macro cleanup in transitive dependencies in ARROW-1452
- * [PARQUET-1074] - [C++] Switch to long key ids in KEYs file
- * [PARQUET-1075] - C++: Coverage upload is broken
- * [PARQUET-1088] - [CPP] remove parquet_version.h from version control since it gets auto generated
- * [PARQUET-1002] - [C++] Compute statistics based on Logical Types
- * [PARQUET-1100] - [C++] Reading repeated types should decode number of records rather than number of values
- * [PARQUET-1090] - [C++] Fix int32 overflow in Arrow table writer, add max row group size property
- * [PARQUET-1108] - [C++] Fix Int96 comparators
-
-## Improvement
- * [PARQUET-1104] - [C++] Upgrade to Apache Arrow 0.7.0 RC0
- * [PARQUET-1072] - [C++] Add ARROW_NO_DEPRECATED_API to CI to check for deprecated API use
- * [PARQUET-1096] - C++: Update sha{1, 256, 512} checksums per latest ASF release policy
- * [PARQUET-1079] - [C++] Account for Arrow API change in ARROW-1335
- * [PARQUET-1087] - [C++] Add wrapper for ScanFileContents in parquet::arrow that catches exceptions
- * [PARQUET-1093] - C++: Improve Arrow level generation error message
- * [PARQUET-1094] - C++: Add benchmark for boolean Arrow column I/O
- * [PARQUET-1083] - [C++] Refactor core logic in parquet-scan.cc so that it can be used as a library function for benchmarking
- * [PARQUET-1037] - Allow final RowGroup to be unfilled
-
-## New Feature
- * [PARQUET-1078] - [C++] Add Arrow writer option to coerce timestamps to milliseconds or microseconds
- * [PARQUET-929] - [C++] Handle arrow::DictionaryArray when writing Arrow data
-
-
-Parquet C++ 1.2.0
---------------------------------------------------------------------------------
-## Bug
- * [PARQUET-1029] - [C++] TypedColumnReader/TypeColumnWriter symbols are no longer being exported
- * [PARQUET-997] - Fix override compiler warnings
- * [PARQUET-1033] - Mismatched Read and Write
- * [PARQUET-1007] - [C++ ] Update parquet.thrift from https://github.com/apache/parquet-format
- * [PARQUET-1039] - PARQUET-911 Breaks Arrow
- * [PARQUET-1038] - Key value metadata should be nullptr if not set
- * [PARQUET-1018] - [C++] parquet.dll has runtime dependencies on one or more libraries in the build toolchain
- * [PARQUET-1003] - [C++] Modify DEFAULT_CREATED_BY value for every new release version
- * [PARQUET-1004] - CPP Building fails on windows
- * [PARQUET-1040] - Missing writer method implementations
- * [PARQUET-1054] - [C++] Account for Arrow API changes in ARROW-1199
- * [PARQUET-1042] - C++: Compilation breaks on GCC 4.8
- * [PARQUET-1048] - [C++] Static linking of libarrow is no longer supported
- * [PARQUET-1013] - Fix ZLIB_INCLUDE_DIR
- * [PARQUET-998] - C++: Release script is not usable
- * [PARQUET-1023] - [C++] Brotli libraries are not being statically linked on Windows
- * [PARQUET-1000] - [C++] Do not build thirdparty Arrow with /WX on MSVC
- * [PARQUET-1052] - [C++] add_compiler_export_flags() throws warning with CMake >= 3.3
- * [PARQUET-1069] - C++: ./dev/release/verify-release-candidate is broken due to missing Arrow dependencies
-
-## Improvement
- * [PARQUET-996] - Improve MSVC build - ThirdpartyToolchain - Arrow
- * [PARQUET-911] - C++: Support nested structs in parquet_arrow
- * [PARQUET-986] - Improve MSVC build - ThirdpartyToolchain - Thrift
- * [PARQUET-864] - [C++] Consolidate non-Parquet-specific bit utility code into Apache Arrow
- * [PARQUET-1043] - [C++] Raise minimum supported CMake version to 3.2
- * [PARQUET-1016] - Upgrade thirdparty Arrow to 0.4.0
- * [PARQUET-858] - [C++] Flatten parquet/column directory, consolidate related code
- * [PARQUET-978] - [C++] Minimizing footer reads for small(ish) metadata
- * [PARQUET-991] - [C++] Fix compiler warnings on MSVC and build with /WX in Appveyor
- * [PARQUET-863] - [C++] Move SIMD, CPU info, hashing, and other generic utilities into Apache Arrow
- * [PARQUET-1053] - Fix unused result warnings due to unchecked Statuses
- * [PARQUET-1067] - C++: Update arrow hash to 0.5.0
- * [PARQUET-1041] - C++: Support Arrow's NullArray
- * [PARQUET-1008] - Update TypedColumnReader::ReadBatch method to accept batch_size as int64_t
- * [PARQUET-1044] - [C++] Use compression libraries from Apache Arrow
- * [PARQUET-999] - Improve MSVC build - Enable PARQUET_BUILD_BENCHMARKS
- * [PARQUET-967] - [C++] Combine libparquet/libparquet_arrow libraries
- * [PARQUET-1045] - [C++] Refactor to account for computational utility code migration in ARROW-1154
-
-## New Feature
- * [PARQUET-1035] - Write Int96 from Arrow Timestamp(ns)
-
-## Task
- * [PARQUET-994] - C++: release-candidate script should not push to master
- * [PARQUET-902] - [C++] Move compressor interfaces into Apache Arrow
-
-## Test
- * [PARQUET-706] - [C++] Create test case that uses libparquet as a 3rd party library
-
-
-Parquet C++ 1.1.0
---------------------------------------------------------------------------------
-## Bug
- * [PARQUET-898] - [C++] Change Travis CI OS X image to Xcode 6.4 and fix our thirdparty build
- * [PARQUET-976] - [C++] Pass unit test suite with MSVC, build in Appveyor
- * [PARQUET-963] - [C++] Disallow reading struct types in Arrow reader for now
- * [PARQUET-959] - [C++] Arrow thirdparty build fails on multiarch systems
- * [PARQUET-962] - [C++] GTEST_MAIN_STATIC_LIB is not defined in FindGTest.cmake
- * [PARQUET-958] - [C++] Print Parquet metadata in JSON format
- * [PARQUET-956] - C++: BUILD_BYPRODUCTS not specified anymore for gtest
- * [PARQUET-948] - [C++] Account for API changes in ARROW-782
- * [PARQUET-947] - [C++] Refactor to account for ARROW-795 Arrow core library consolidation
- * [PARQUET-965] - [C++] FIXED_LEN_BYTE_ARRAY types are unhandled in the Arrow reader
- * [PARQUET-949] - [C++] Arrow version pinning seems to not be working properly
- * [PARQUET-955] - [C++] pkg_check_modules will override $ARROW_HOME if it is set in the environment
- * [PARQUET-945] - [C++] Thrift static libraries are not used with recent patch
- * [PARQUET-943] - [C++] Overflow build error on x86
- * [PARQUET-938] - [C++] There is a typo in cmake_modules/FindSnappy.cmake comment
- * [PARQUET-936] - [C++] parquet::arrow::WriteTable can enter infinite loop if chunk_size is 0
- * [PARQUET-981] - Repair usage of *_HOME 3rd party dependencies environment variables during Windows build
- * [PARQUET-992] - [C++] parquet/compression.h leaks zlib.h
- * [PARQUET-987] - [C++] Fix regressions caused by PARQUET-981
- * [PARQUET-933] - [C++] Account for Arrow Table API changes coming in ARROW-728
- * [PARQUET-915] - Support Arrow Time Types in Schema
- * [PARQUET-914] - [C++] Throw more informative exception when user writes too many values to a column in a row group
- * [PARQUET-923] - [C++] Account for Time metadata changes in ARROW-686
- * [PARQUET-918] - FromParquetSchema API crashes on nested schemas
- * [PARQUET-925] - [C++] FindArrow.cmake sets the wrong library path after ARROW-648
- * [PARQUET-932] - [c++] Add option to build parquet library with minimal dependency
- * [PARQUET-919] - [C++] Account for API changes in ARROW-683
- * [PARQUET-995] - [C++] Int96 reader in parquet_arrow uses size of Int96Type instead of Int96
-
-## Improvement
- * [PARQUET-508] - Add ParquetFilePrinter
- * [PARQUET-595] - Add API for key-value metadata
- * [PARQUET-897] - [C++] Only use designated public headers from libarrow
- * [PARQUET-679] - [C++] Build and unit tests support for MSVC on Windows
- * [PARQUET-977] - Improve MSVC build
- * [PARQUET-957] - [C++] Add optional $PARQUET_BUILD_TOOLCHAIN environment variable option for configuring build environment
- * [PARQUET-961] - [C++] Strip debug symbols from libparquet libraries in release builds by default
- * [PARQUET-954] - C++: Use Brolti 0.6 release
- * [PARQUET-953] - [C++] Change arrow::FileWriter API to be initialized from a Schema, and provide for writing multiple tables
- * [PARQUET-941] - [C++] Stop needless Boost static library detection for CentOS 7 support
- * [PARQUET-942] - [C++] Fix wrong variabe use in FindSnappy
- * [PARQUET-939] - [C++] Support Thrift_HOME CMake variable like FindSnappy does as Snappy_HOME
- * [PARQUET-940] - [C++] Fix Arrow library path detection
- * [PARQUET-937] - [C++] Support CMake < 3.4 again for Arrow detection
- * [PARQUET-935] - [C++] Set shared library version for .deb packages
- * [PARQUET-934] - [C++] Support multiarch on Debian
- * [PARQUET-984] - C++: Add abi and so version to pkg-config
- * [PARQUET-983] - C++: Update Thirdparty hash to Arrow 0.3.0
- * [PARQUET-989] - [C++] Link dynamically to libarrow in toolchain build, set LD_LIBRARY_PATH
- * [PARQUET-988] - [C++] Add Linux toolchain-based build to Travis CI
- * [PARQUET-928] - [C++] Support pkg-config
- * [PARQUET-927] - [C++] Specify shared library version of Apache Arrow
- * [PARQUET-931] - [C++] Add option to pin thirdparty Arrow version used in ExternalProject
- * [PARQUET-926] - [C++] Use pkg-config to find Apache Arrow
- * [PARQUET-917] - C++: Build parquet_arrow by default
- * [PARQUET-910] - C++: Support TIME logical type in parquet_arrow
- * [PARQUET-909] - [CPP]: Reduce buffer allocations (mallocs) on critical path
-
-## New Feature
- * [PARQUET-853] - [C++] Add option to link with shared boost libraries when building Arrow in the thirdparty toolchain
- * [PARQUET-946] - [C++] Refactoring in parquet::arrow::FileReader to be able to read a single row group
- * [PARQUET-930] - [C++] Account for all Arrow date/time types
-
-
-Parquet C++ 1.0.0
---------------------------------------------------------------------------------
-## Bug
- * [PARQUET-455] - Fix compiler warnings on OS X / Clang
- * [PARQUET-558] - Support ZSH in build scripts
- * [PARQUET-720] - Parquet-cpp fails to link when included in multiple TUs
- * [PARQUET-718] - Reading boolean pages written by parquet-cpp fails
- * [PARQUET-640] - [C++] Force the use of gcc 4.9 in conda builds
- * [PARQUET-643] - Add const modifier to schema pointer reference in ParquetFileWriter
- * [PARQUET-672] - [C++] Build testing conda artifacts in debug mode
- * [PARQUET-661] - [C++] Do not assume that perl is found in /usr/bin
- * [PARQUET-659] - [C++] Instantiated template visibility is broken on clang / OS X
- * [PARQUET-657] - [C++] Don't define DISALLOW_COPY_AND_ASSIGN if already defined
- * [PARQUET-656] - [C++] Revert PARQUET-653
- * [PARQUET-676] - MAX_VALUES_PER_LITERAL_RUN causes RLE encoding failure
- * [PARQUET-614] - C++: Remove unneeded LZ4-related code
- * [PARQUET-604] - Install writer.h headers
- * [PARQUET-621] - C++: Uninitialised DecimalMetadata is read
- * [PARQUET-620] - C++: Duplicate calls to ParquetFileWriter::Close cause duplicate metdata writes
- * [PARQUET-599] - ColumnWriter::RleEncodeLevels' size estimation might be wrong
- * [PARQUET-617] - C++: Enable conda build to work on systems with non-default C++ toolchains
- * [PARQUET-627] - Ensure that thrift headers are generated before source compilation
- * [PARQUET-745] - TypedRowGroupStatistics fails to PlainDecode min and max in ByteArrayType
- * [PARQUET-738] - Update arrow version that also supports newer Xcode
- * [PARQUET-747] - [C++] TypedRowGroupStatistics are not being exported in libparquet.so
- * [PARQUET-711] - Use metadata builders in parquet writer
- * [PARQUET-732] - Building a subset of dependencies does not work
- * [PARQUET-760] - On switching from dictionary to the fallback encoding, an incorrect encoding is set
- * [PARQUET-691] - [C++] Write ColumnChunk metadata after each column chunk in the file
- * [PARQUET-797] - [C++] Update for API changes in ARROW-418
- * [PARQUET-837] - [C++] SerializedFile::ParseMetaData uses Seek, followed by Read, and could have race conditions
- * [PARQUET-827] - [C++] Incorporate addition of arrow::MemoryPool::Reallocate
- * [PARQUET-502] - Scanner segfaults when its batch size is smaller than the number of rows
- * [PARQUET-469] - Roll back Thrift bindings to 0.9.0
- * [PARQUET-889] - Fix compilation when PARQUET_USE_SSE is on
- * [PARQUET-888] - C++ Memory leak in RowGroupSerializer
- * [PARQUET-819] - C++: Trying to install non-existing parquet/arrow/utils.h
- * [PARQUET-736] - XCode 8.0 breaks builds
- * [PARQUET-505] - Column reader: automatically handle large data pages
- * [PARQUET-615] - C++: Building static or shared libparquet should not be mutually exclusive
- * [PARQUET-658] - ColumnReader has no virtual destructor
- * [PARQUET-799] - concurrent usage of the file reader API
- * [PARQUET-513] - Valgrind errors are not failing the Travis CI build
- * [PARQUET-841] - [C++] Writing wrong format version when using ParquetVersion::PARQUET_1_0
- * [PARQUET-742] - Add missing license headers
- * [PARQUET-741] - compression_buffer_ is reused although it shouldn't
- * [PARQUET-700] - C++: Disable dictionary encoding for boolean columns
- * [PARQUET-662] - [C++] ParquetException must be explicitly exported in dynamic libraries
- * [PARQUET-704] - [C++] scan-all.h is not being installed
- * [PARQUET-865] - C++: Pass all CXXFLAGS to Thrift ExternalProject
- * [PARQUET-875] - [C++] Fix coveralls build given changes to thirdparty build procedure
- * [PARQUET-709] - [C++] Fix conda dev binary builds
- * [PARQUET-638] - [C++] Revert static linking of libstdc++ in conda builds until symbol visibility addressed
- * [PARQUET-606] - Travis coverage is broken
- * [PARQUET-880] - [CPP] Prevent destructors from throwing
- * [PARQUET-886] - [C++] Revise build documentation and requirements in README.md
- * [PARQUET-900] - C++: Fix NOTICE / LICENSE issues
- * [PARQUET-885] - [C++] Do not search for Thrift in default system paths
- * [PARQUET-879] - C++: ExternalProject compilation for Thrift fails on older CMake versions
- * [PARQUET-635] - [C++] Statically link libstdc++ on Linux in conda recipe
- * [PARQUET-710] - Remove unneeded private member variables from RowGroupReader ABI
- * [PARQUET-766] - C++: Expose ParquetFileReader through Arrow reader as const
- * [PARQUET-876] - C++: Correct snapshot version
- * [PARQUET-821] - [C++] zlib download link is broken
- * [PARQUET-818] - [C++] Refactor library to share IO, Buffer, and memory management abstractions with Apache Arrow
- * [PARQUET-537] - LocalFileSource leaks resources
- * [PARQUET-764] - [CPP] Parquet Writer does not write Boolean values correctly
- * [PARQUET-812] - [C++] Failure reading BYTE_ARRAY data from file in parquet-compatibility project
- * [PARQUET-759] - Cannot store columns consisting of empty strings
- * [PARQUET-846] - [CPP] CpuInfo::Init() is not thread safe
- * [PARQUET-694] - C++: Revert default data page size back to 1M
- * [PARQUET-842] - [C++] Impala rejects DOUBLE columns if decimal metadata is set
- * [PARQUET-708] - [C++] RleEncoder does not account for "worst case scenario" in MaxBufferSize for bit_width > 1
- * [PARQUET-639] - Do not export DCHECK in public headers
- * [PARQUET-828] - [C++] "version" field set improperly in file metadata
- * [PARQUET-891] - [C++] Do not search for Snappy in default system paths
- * [PARQUET-626] - Fix builds due to unavailable llvm.org apt mirror
- * [PARQUET-629] - RowGroupSerializer should only close itself once
- * [PARQUET-472] - Clean up InputStream ownership semantics in ColumnReader
- * [PARQUET-739] - Rle-decoding uses static buffer that is shared accross threads
- * [PARQUET-561] - ParquetFileReader::Contents PIMPL missing a virtual destructor
- * [PARQUET-892] - [C++] Clean up link library targets in CMake files
- * [PARQUET-454] - Address inconsistencies in boolean decoding
- * [PARQUET-816] - [C++] Failure decoding sample dict-encoded file from parquet-compatibility project
- * [PARQUET-565] - Use PATH instead of DIRECTORY in get_filename_component to support CMake<2.8.12
- * [PARQUET-446] - Hide thrift dependency in parquet-cpp
- * [PARQUET-843] - [C++] Impala unable to read files created by parquet-cpp
- * [PARQUET-555] - Dictionary page metadata handling inconsistencies
- * [PARQUET-908] - Fix for PARQUET-890 introduces undefined symbol in libparquet_arrow.so
- * [PARQUET-793] - [CPP] Do not return incorrect statistics
- * [PARQUET-887] - C++: Fix issues in release scripts arise in RC1
-
-## Improvement
- * [PARQUET-277] - Remove boost dependency
- * [PARQUET-500] - Enable coveralls.io for apache/parquet-cpp
- * [PARQUET-497] - Decouple Parquet physical file structure from FileReader class
- * [PARQUET-597] - Add data rates to benchmark output
- * [PARQUET-522] - #include cleanup with include-what-you-use
- * [PARQUET-515] - Add "Reset" to LevelEncoder and LevelDecoder
- * [PARQUET-514] - Automate coveralls.io updates in Travis CI
- * [PARQUET-551] - Handle compiler warnings due to disabled DCHECKs in release builds
- * [PARQUET-559] - Enable InputStream as a source to the ParquetFileReader
- * [PARQUET-562] - Simplified ZSH support in build scripts
- * [PARQUET-538] - Improve ColumnReader Tests
- * [PARQUET-541] - Portable build scripts
- * [PARQUET-724] - Test more advanced properties setting
- * [PARQUET-641] - Instantiate stringstream only if needed in SerializedPageReader::NextPage
- * [PARQUET-636] - Expose selection for different encodings
- * [PARQUET-603] - Implement missing information in schema descriptor
- * [PARQUET-610] - Print ColumnMetaData for each RowGroup
- * [PARQUET-600] - Add benchmarks for RLE-Level encoding
- * [PARQUET-592] - Support compressed writes
- * [PARQUET-593] - Add API for writing Page statistics
- * [PARQUET-589] - Implement Chunked InMemoryInputStream for better memory usage
- * [PARQUET-587] - Implement BufferReader::Read(int64_t,uint8_t*)
- * [PARQUET-616] - C++: WriteBatch should accept const arrays
- * [PARQUET-630] - C++: Support link flags for older CMake versions
- * [PARQUET-634] - Consistent private linking of dependencies
- * [PARQUET-633] - Add version to WriterProperties
- * [PARQUET-625] - Improve RLE read performance
- * [PARQUET-737] - Use absolute namespace in macros
- * [PARQUET-762] - C++: Use optimistic allocation instead of Arrow Builders
- * [PARQUET-773] - C++: Check licenses with RAT in CI
- * [PARQUET-687] - C++: Switch to PLAIN encoding if dictionary grows too large
- * [PARQUET-784] - C++: Reference Spark, Kudu and FrameOfReference in LICENSE
- * [PARQUET-809] - [C++] Add API to determine if two files' schemas are compatible
- * [PARQUET-778] - Standardize the schema output to match the parquet-mr format
- * [PARQUET-463] - Add DCHECK* macros for assertions in debug builds
- * [PARQUET-471] - Use the same environment setup script for Travis CI as local sandbox development
- * [PARQUET-449] - Update to latest parquet.thrift
- * [PARQUET-496] - Fix cpplint configuration to be more restrictive
- * [PARQUET-468] - Add a cmake option to generate the Parquet thrift headers with the thriftc in the environment
- * [PARQUET-482] - Organize src code file structure to have a very clear folder with public headers.
- * [PARQUET-591] - Page size estimation during writes
- * [PARQUET-518] - Review usages of size_t and unsigned integers generally per Google style guide
- * [PARQUET-533] - Simplify RandomAccessSource API to combine Seek/Read
- * [PARQUET-767] - Add release scripts for parquet-cpp
- * [PARQUET-699] - Update parquet.thrift from https://github.com/apache/parquet-format
- * [PARQUET-653] - [C++] Re-enable -static-libstdc++ in dev artifact builds
- * [PARQUET-763] - C++: Expose ParquetFileReader through Arrow reader
- * [PARQUET-857] - [C++] Flatten parquet/encodings directory
- * [PARQUET-862] - Provide defaut cache size values if CPU info probing is not available
- * [PARQUET-689] - C++: Compress DataPages eagerly
- * [PARQUET-874] - [C++] Use default memory allocator from Arrow
- * [PARQUET-267] - Detach thirdparty code from build configuration.
- * [PARQUET-418] - Add a utility to print contents of a Parquet file to stdout
- * [PARQUET-519] - Disable compiler warning supressions and fix all DEBUG build warnings
- * [PARQUET-447] - Add Debug and Release build types and associated compiler flags
- * [PARQUET-868] - C++: Build snappy with optimizations
- * [PARQUET-894] - Fix compilation warning
- * [PARQUET-883] - C++: Support non-standard gcc version strings
- * [PARQUET-607] - Public Writer header
- * [PARQUET-731] - [CPP] Add API to return metadata size and Skip reading values
- * [PARQUET-628] - Link thrift privately
- * [PARQUET-877] - C++: Update Arrow Hash, update Version in metadata.
- * [PARQUET-547] - Refactor most templates to use DataType structs rather than the Type::type enum
- * [PARQUET-882] - [CPP] Improve Application Version parsing
- * [PARQUET-448] - Add cmake option to skip building the unit tests
- * [PARQUET-721] - Performance benchmarks for reading into Arrow structures
- * [PARQUET-820] - C++: Decoders should directly emit arrays with spacing for null entries
- * [PARQUET-813] - C++: Build dependencies using CMake External project
- * [PARQUET-488] - Add SSE-related cmake options to manage compiler flags
- * [PARQUET-564] - Add option to run unit tests with valgrind --tool=memcheck
- * [PARQUET-572] - Rename parquet_cpp namespace to parquet
- * [PARQUET-829] - C++: Make use of ARROW-469
- * [PARQUET-501] - Add an OutputStream abstraction (capable of memory allocation) for Encoder public API
- * [PARQUET-744] - Clarifications on build instructions
- * [PARQUET-520] - Add version of LocalFileSource that uses memory-mapping for zero-copy reads
- * [PARQUET-556] - Extend RowGroupStatistics to include "min" "max" statistics
- * [PARQUET-671] - Improve performance of RLE/bit-packed decoding in parquet-cpp
- * [PARQUET-681] - Add tool to scan a parquet file
-
-## New Feature
- * [PARQUET-499] - Complete PlainEncoder implementation for all primitive types and test end to end
- * [PARQUET-439] - Conform all copyright headers to ASF requirements
- * [PARQUET-436] - Implement ParquetFileWriter class entry point for generating new Parquet files
- * [PARQUET-435] - Provide vectorized ColumnReader interface
- * [PARQUET-438] - Update RLE encoder/decoder modules from Impala upstream changes and adapt unit tests
- * [PARQUET-512] - Add optional google/benchmark 3rd-party dependency for performance testing
- * [PARQUET-566] - Add method to retrieve the full column path
- * [PARQUET-613] - C++: Add conda packaging recipe
- * [PARQUET-605] - Expose schema node in ColumnDescriptor
- * [PARQUET-619] - C++: Add OutputStream for local files
- * [PARQUET-583] - Implement Parquet to Thrift schema conversion
- * [PARQUET-582] - Conversion functions for Parquet enums to Thrift enums
- * [PARQUET-728] - [C++] Bring parquet::arrow up to date with API changes in arrow::io
- * [PARQUET-752] - [C++] Conform parquet_arrow to upstream API changes
- * [PARQUET-788] - [C++] Reference Impala / Apache Impala (incubating) in LICENSE
- * [PARQUET-808] - [C++] Add API to read file given externally-provided FileMetadata
- * [PARQUET-807] - [C++] Add API to read file metadata only from a file handle
- * [PARQUET-805] - C++: Read Int96 into Arrow Timestamp(ns)
- * [PARQUET-836] - [C++] Add column selection to parquet::arrow::FileReader
- * [PARQUET-835] - [C++] Add option to parquet::arrow to read columns in parallel using a thread pool
- * [PARQUET-830] - [C++] Add additional configuration options to parquet::arrow::OpenFIle
- * [PARQUET-769] - C++: Add support for Brotli Compression
- * [PARQUET-489] - Add visibility macros to be used for public and internal APIs of libparquet
- * [PARQUET-542] - Support memory allocation from external memory
- * [PARQUET-844] - [C++] Consolidate encodings, schema, and compression subdirectories into fewer files
- * [PARQUET-848] - [C++] Consolidate libparquet_thrift subcomponent
- * [PARQUET-646] - [C++] Enable easier 3rd-party toolchain clang builds on Linux
- * [PARQUET-598] - [C++] Test writing all primitive data types
- * [PARQUET-442] - Convert flat SchemaElement vector to implied nested schema data structure
- * [PARQUET-867] - [C++] Support writing sliced Arrow arrays
- * [PARQUET-456] - Add zlib codec support
- * [PARQUET-834] - C++: Support r/w of arrow::ListArray
- * [PARQUET-485] - Decouple data page delimiting from column reader / scanner classes, create test fixtures
- * [PARQUET-434] - Add a ParquetFileReader class to encapsulate some low-level details of interacting with Parquet files
- * [PARQUET-666] - PLAIN_DICTIONARY write support
- * [PARQUET-437] - Incorporate googletest thirdparty dependency and add cmake tools (ADD_PARQUET_TEST) to simplify adding new unit tests
- * [PARQUET-866] - [C++] Account for API changes in ARROW-33
- * [PARQUET-545] - Improve API to support Decimal type
- * [PARQUET-579] - Add API for writing Column statistics
- * [PARQUET-494] - Implement PLAIN_DICTIONARY encoding and decoding
- * [PARQUET-618] - C++: Automatically upload conda build artifacts on commits to master
- * [PARQUET-833] - C++: Provide API to write spaced arrays (e.g. Arrow)
- * [PARQUET-903] - C++: Add option to set RPATH to ORIGIN
- * [PARQUET-451] - Add a RowGroup reader interface class
- * [PARQUET-785] - C++: List conversion for Arrow Schemas
- * [PARQUET-712] - C++: Read into Arrow memory
- * [PARQUET-890] - C++: Support I/O of DATE columns in parquet_arrow
- * [PARQUET-782] - C++: Support writing to Arrow sinks
- * [PARQUET-849] - [C++] Upgrade default Thrift in thirdparty toolchain to 0.9.3 or 0.10
- * [PARQUET-573] - C++: Create a public API for reading and writing file metadata
-
-## Task
- * [PARQUET-814] - C++: Remove Conda recipes
- * [PARQUET-503] - Re-enable parquet 2.0 encodings
- * [PARQUET-169] - Parquet-cpp: Implement support for bulk reading and writing repetition/definition levels.
- * [PARQUET-878] - C++: Remove setup_build_env from rc-verification script
- * [PARQUET-881] - C++: Update Arrow hash to 0.2.0-rc2
- * [PARQUET-771] - C++: Sync KEYS file
- * [PARQUET-901] - C++: Publish RCs in apache-parquet-VERSION in SVN
-
-## Test
- * [PARQUET-525] - Test coverage for malformed file failure modes on the read path
- * [PARQUET-703] - [C++] Validate num_values metadata for columns with nulls
- * [PARQUET-507] - Improve runtime of rle-test.cc
- * [PARQUET-549] - Add scanner and column reader tests for dictionary data pages
- * [PARQUET-457] - Add compressed data page unit tests
-
-
diff --git a/CMakeLists.txt b/CMakeLists.txt
deleted file mode 100644
index fa4fb21..0000000
--- a/CMakeLists.txt
+++ /dev/null
@@ -1,783 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-# Require cmake that supports BYPRODUCTS in add_custom_command, ExternalProject_Add [1].
-cmake_minimum_required(VERSION 3.2.0)
-
-message(FATAL_ERROR "Building parquet-cpp master is no longer supported. "
- "Build from C++ codebase in https://github.com/apache/arrow with -DARROW_PARQUET=ON")
-
-file(READ "${CMAKE_CURRENT_SOURCE_DIR}/.parquetcppversion" PARQUET_VERSION)
-string(REPLACE "\n" "" PARQUET_VERSION "${PARQUET_VERSION}")
-string(REGEX MATCH "^([0-9]+\.[0-9]+\.[0-9]+(\.[0-9]+)?)" VERSION ${PARQUET_VERSION})
-if(NOT VERSION)
- message(FATAL_ERROR "invalid .parquetcppversion")
-endif()
-
-project(parquet-cpp VERSION ${VERSION})
-set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "C++ library to read and write the Apache Parquet columnar data format")
-set(CPACK_PACKAGE_VERSION_MAJOR ${PROJECT_VERSION_MAJOR})
-set(CPACK_PACKAGE_VERSION_MINOR ${PROJECT_VERSION_MINOR})
-set(CPACK_PACKAGE_VERSION_PATCH ${PROJECT_VERSION_PATCH})
-set(CPACK_PACKAGE_VENDOR "Apache Software Foundation")
-set(CPACK_PACKAGE_CONTACT "Apache Parquet Development <dev@parquet.apache.org>")
-set(CPACK_STRIP_FILES TRUE)
-set(CPACK_RESOURCE_FILE_LICENSE "${CMAKE_SOURCE_DIR}/LICENSE.txt")
-set(CPACK_DEBIAN_FILE_NAME "DEB-DEFAULT")
-set(CPACK_RPM_FILE_NAME "RPM-DEFAULT")
-set(CPACK_RPM_PACKAGE_LICENSE "Apache v2.0")
-
-include(ExternalProject)
-include(FindPkgConfig)
-
-# This ensures that things like gnu++11 get passed correctly
-set(CMAKE_CXX_STANDARD 11)
-
-# We require a C++11 compliant compiler
-set(CMAKE_CXX_STANDARD_REQUIRED ON)
-
-include(GNUInstallDirs)
-
-set(PARQUET_SO_VERSION ${PROJECT_VERSION_MAJOR})
-set(PARQUET_ABI_VERSION ${PROJECT_VERSION})
-if(WIN32)
- set(INSTALL_CMAKE_DIR cmake)
-else()
- set(INSTALL_CMAKE_DIR "${CMAKE_INSTALL_DATAROOTDIR}/${PROJECT_NAME}/cmake")
-endif()
-set(CPACK_RPM_EXCLUDE_FROM_AUTO_FILELIST_ADDITION "/usr/${CMAKE_INSTALL_LIBDIR}/pkgconfig")
-include(CPack)
-
-include(CMakePackageConfigHelpers)
-configure_package_config_file(cmake_modules/${PROJECT_NAME}Config.cmake.in
- ${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}Config.cmake
- INSTALL_DESTINATION ${INSTALL_CMAKE_DIR}
- PATH_VARS CMAKE_INSTALL_INCLUDEDIR CMAKE_INSTALL_LIBDIR)
-write_basic_package_version_file(${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}ConfigVersion.cmake
- COMPATIBILITY SameMajorVersion)
-install(FILES ${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}Config.cmake
- ${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}ConfigVersion.cmake
-DESTINATION ${INSTALL_CMAKE_DIR})
-
-if (NOT "$ENV{PARQUET_GCC_ROOT}" STREQUAL "")
- set(GCC_ROOT $ENV{PARQUET_GCC_ROOT})
- set(CMAKE_C_COMPILER ${GCC_ROOT}/bin/gcc)
- set(GCOV_PATH ${GCC_ROOT}/bin/gcov)
- set(CMAKE_CXX_COMPILER ${GCC_ROOT}/bin/g++)
-endif()
-
-# generate CTest input files
-enable_testing()
-
-# where to find cmake modules
-set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/cmake_modules")
-set(BUILD_SUPPORT_DIR "${CMAKE_SOURCE_DIR}/build-support")
-
-set(CLANG_FORMAT_VERSION "6.0")
-find_package(ClangTools)
-if ("$ENV{CMAKE_EXPORT_COMPILE_COMMANDS}" STREQUAL "1" OR CLANG_TIDY_FOUND)
- # Generate a Clang compile_commands.json "compilation database" file for use
- # with various development tools, such as Vim's YouCompleteMe plugin.
- # See http://clang.llvm.org/docs/JSONCompilationDatabase.html
- set(CMAKE_EXPORT_COMPILE_COMMANDS 1)
-endif()
-
-find_program(CCACHE_FOUND ccache)
-if(CCACHE_FOUND)
- set_property(GLOBAL PROPERTY RULE_LAUNCH_COMPILE ccache)
- set_property(GLOBAL PROPERTY RULE_LAUNCH_LINK ccache)
-endif(CCACHE_FOUND)
-
-if(APPLE)
- set(CMAKE_MACOSX_RPATH 1)
-endif()
-
-# if no build build type is specified, default to debug builds
-if (NOT CMAKE_BUILD_TYPE)
- set(CMAKE_BUILD_TYPE Debug)
-endif(NOT CMAKE_BUILD_TYPE)
-
-# set compile output directory
-string (TOLOWER ${CMAKE_BUILD_TYPE} BUILD_SUBDIR_NAME)
-
-# Top level cmake file, set options
-if ("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}")
- option(PARQUET_BUILD_SHARED
- "Build the shared version of libparquet"
- ON)
- option(PARQUET_BUILD_STATIC
- "Build the static version of libparquet. Always ON if building unit tests"
- ON)
- set(PARQUET_ARROW_LINKAGE "shared" CACHE STRING
- "Libraries to link for Apache Arrow. static|shared (default shared)")
- set(PARQUET_CXXFLAGS "" CACHE STRING
- "Compiler flags to use when compiling Parquet")
- option(PARQUET_USE_SSE
- "Build with SSE4 optimizations"
- OFF)
- option(PARQUET_BUILD_BENCHMARKS
- "Build the libparquet benchmark suite"
- OFF)
-
- set(PARQUET_BUILD_WARNING_LEVEL "PRODUCTION" CACHE STRING
- "Levels of compiler warnings for development: PRODUCTION/CHECKIN/EVERYTHING")
-
- option(PARQUET_BOOST_USE_SHARED
- "Rely on boost shared libraries where relevant"
- ON)
- option(PARQUET_BUILD_TESTS
- "Build the libparquet test suite"
- ON)
- option(PARQUET_TEST_MEMCHECK
- "Run the test suite using valgrind --tool=memcheck"
- OFF)
- option(PARQUET_BUILD_EXECUTABLES
- "Build the libparquet executable CLI tools"
- ON)
- option(PARQUET_RPATH_ORIGIN
- "Build Parquet libraries with RPATH set to \$ORIGIN"
- OFF)
- option(PARQUET_MINIMAL_DEPENDENCY
- "Depend only on Thirdparty headers to build libparquet. Always OFF if building binaries"
- OFF)
-
- option(PARQUET_THRIFT_USE_BOOST
- "Enable if Thirdparty Thrift uses boost::shared_ptr (Apache Thrift < 0.11)"
- OFF)
-
- if (MSVC)
- set(ARROW_MSVC_STATIC_LIB_SUFFIX "_static" CACHE STRING
- "Arrow static lib suffix used on Windows with MSVC (default _static)")
- set(THRIFT_MSVC_STATIC_LIB_SUFFIX "md" CACHE STRING
- "Thrift static lib suffix used on Windows with MSVC (default md)")
- option(PARQUET_USE_STATIC_CRT
- "Build Parquet with statically linked CRT"
- OFF)
- option(PARQUET_USE_CLCACHE
- "Use clcache if available"
- ON)
- endif()
-
- option(PARQUET_VERBOSE_THIRDPARTY_BUILD
- "If off, output from ExternalProjects will be logged to files rather than shown"
- OFF)
-
-endif()
-
-if (MSVC AND PARQUET_USE_CLCACHE AND
- (("${CMAKE_GENERATOR}" STREQUAL "NMake Makefiles") OR
- ("${CMAKE_GENERATOR}" STREQUAL "Ninja")))
- find_program(CLCACHE_FOUND clcache)
- if(CLCACHE_FOUND)
- set(CMAKE_CXX_COMPILER ${CLCACHE_FOUND})
- endif(CLCACHE_FOUND)
-endif()
-
-include(BuildUtils)
-
-if (PARQUET_BUILD_TESTS OR PARQUET_BUILD_EXECUTABLES OR PARQUET_BUILD_BENCHMARKS)
- set(PARQUET_BUILD_STATIC ON)
- set(PARQUET_MINIMAL_DEPENDENCY OFF)
-endif()
-
-# If build in-source, create the latest symlink. If build out-of-source, which is
-# preferred, simply output the binaries in the build folder
-if (${CMAKE_SOURCE_DIR} STREQUAL "${CMAKE_CURRENT_BINARY_DIR}")
- set(BUILD_OUTPUT_ROOT_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/build/${BUILD_SUBDIR_NAME}")
- # Link build/latest to the current build directory, to avoid developers
- # accidentally running the latest debug build when in fact they're building
- # release builds.
- FILE(MAKE_DIRECTORY ${BUILD_OUTPUT_ROOT_DIRECTORY})
- if (NOT APPLE)
- set(MORE_ARGS "-T")
- endif()
-EXECUTE_PROCESS(COMMAND ln ${MORE_ARGS} -sf ${BUILD_OUTPUT_ROOT_DIRECTORY}
- ${CMAKE_CURRENT_BINARY_DIR}/build/latest)
-else()
- set(BUILD_OUTPUT_ROOT_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/${BUILD_SUBDIR_NAME}")
-endif()
-
-# where to put generated archives (.a files)
-set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY "${BUILD_OUTPUT_ROOT_DIRECTORY}")
-set(ARCHIVE_OUTPUT_DIRECTORY "${BUILD_OUTPUT_ROOT_DIRECTORY}")
-
-# where to put generated libraries (.so files)
-set(CMAKE_LIBRARY_OUTPUT_DIRECTORY "${BUILD_OUTPUT_ROOT_DIRECTORY}")
-set(LIBRARY_OUTPUT_DIRECTORY "${BUILD_OUTPUT_ROOT_DIRECTORY}")
-
-# where to put generated binaries
-set(EXECUTABLE_OUTPUT_PATH "${BUILD_OUTPUT_ROOT_DIRECTORY}")
-
-if (MSVC)
- # define output directories to overwrite default Visual Studio generator output directories
- FOREACH(BUILD_CONFIG ${CMAKE_CONFIGURATION_TYPES})
- string(TOUPPER ${BUILD_CONFIG} UPPERCASE_BUILD_CONFIG)
- set(CMAKE_LIBRARY_OUTPUT_DIRECTORY_${UPPERCASE_BUILD_CONFIG} ${BUILD_OUTPUT_ROOT_DIRECTORY})
- set(CMAKE_PDB_OUTPUT_DIRECTORY_${UPPERCASE_BUILD_CONFIG} ${BUILD_OUTPUT_ROOT_DIRECTORY})
- set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY_${UPPERCASE_BUILD_CONFIG} ${BUILD_OUTPUT_ROOT_DIRECTORY})
- set(CMAKE_RUNTIME_OUTPUT_DIRECTORY_${UPPERCASE_BUILD_CONFIG} ${BUILD_OUTPUT_ROOT_DIRECTORY})
- ENDFOREACH()
-endif()
-
-############################################################
-# Benchmarking
-############################################################
-# Add a new micro benchmark, with or without an executable that should be built.
-# If benchmarks are enabled then they will be run along side unit tests with ctest.
-# 'make runbenchmark' and 'make unittest' to build/run only benchmark or unittests,
-# respectively.
-#
-# REL_BENCHMARK_NAME is the name of the benchmark app. It may be a single component
-# (e.g. monotime-benchmark) or contain additional components (e.g.
-# net/net_util-benchmark). Either way, the last component must be a globally
-# unique name.
-
-# The benchmark will registered as unit test with ctest with a label
-# of 'benchmark'.
-#
-# Arguments after the test name will be passed to set_tests_properties().
-function(ADD_PARQUET_BENCHMARK REL_BENCHMARK_NAME)
- if(NOT PARQUET_BUILD_BENCHMARKS)
- return()
- endif()
- get_filename_component(BENCHMARK_NAME ${REL_BENCHMARK_NAME} NAME_WE)
-
- if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${REL_BENCHMARK_NAME}.cc)
- # This benchmark has a corresponding .cc file, set it up as an executable.
- set(BENCHMARK_PATH "${EXECUTABLE_OUTPUT_PATH}/${BENCHMARK_NAME}")
- add_executable(${BENCHMARK_NAME} "${REL_BENCHMARK_NAME}.cc")
-
- if(APPLE)
- # On OS X / Thrift >= 0.9.2, tr1/tuple.h is not in libc++
- SET_TARGET_PROPERTIES(${TEST_NAME} PROPERTIES COMPILE_FLAGS
- -DGTEST_USE_OWN_TR1_TUPLE=1)
- else()
- # Linux, for Thrift >= 0.9.2
- SET_TARGET_PROPERTIES(${TEST_NAME} PROPERTIES COMPILE_FLAGS
- -DGTEST_USE_OWN_TR1_TUPLE=0)
- endif()
-
- target_link_libraries(${BENCHMARK_NAME} ${PARQUET_BENCHMARK_LINK_LIBS})
- add_dependencies(runbenchmark ${BENCHMARK_NAME})
- set(NO_COLOR "--color_print=false")
- else()
- # No executable, just invoke the benchmark (probably a script) directly.
- set(BENCHMARK_PATH ${CMAKE_CURRENT_SOURCE_DIR}/${REL_BENCHMARK_NAME})
- set(NO_COLOR "")
- endif()
-
- if(WIN32)
- add_test(${BENCHMARK_NAME} ${BENCHMARK_PATH} ${NO_COLOR})
- else()
- add_test(${BENCHMARK_NAME}
- ${BUILD_SUPPORT_DIR}/run-test.sh ${CMAKE_BINARY_DIR} benchmark ${BENCHMARK_PATH} ${NO_COLOR})
- endif()
- set_tests_properties(${BENCHMARK_NAME} PROPERTIES LABELS "benchmark")
- if(ARGN)
- set_tests_properties(${BENCHMARK_NAME} PROPERTIES ${ARGN})
- endif()
-endfunction()
-
-# A wrapper for add_dependencies() that is compatible with NO_BENCHMARKS.
-function(ADD_PARQUET_BENCHMARK_DEPENDENCIES REL_BENCHMARK_NAME)
- if(NOT PARQUET_BUILD_BENCHMARKS)
- return()
- endif()
- get_filename_component(BENCMARK_NAME ${REL_BENCHMARK_NAME} NAME_WE)
-
- add_dependencies(${BENCHMARK_NAME} ${ARGN})
-endfunction()
-
-############################################################
-# Testing
-############################################################
-
-# Add a new test case, with or without an executable that should be built.
-#
-# REL_TEST_NAME is the name of the test. It may be a single component
-# (e.g. monotime-test) or contain additional components (e.g.
-# net/net_util-test). Either way, the last component must be a globally
-# unique name.
-#
-# The unit test is added with a label of "unittest" to support filtering with
-# ctest.
-#
-# Arguments after the test name will be passed to set_tests_properties().
-function(ADD_PARQUET_TEST REL_TEST_NAME)
- set(options)
- set(one_value_args LINKAGE)
- set(multi_value_args)
- cmake_parse_arguments(ARG "${options}" "${one_value_args}" "${multi_value_args}" ${ARGN})
-
- if(NOT PARQUET_BUILD_TESTS)
- return()
- endif()
-
- # TODO(wesm): not very rigorous error checking
- if (ARG_LINKAGE AND "${ARG_LINKAGE}" STREQUAL "shared")
- if(NOT PARQUET_BUILD_SHARED)
- # Skip this test if we are not building the shared library
- return()
- else()
- set(TEST_LINK_LIBS ${PARQUET_TEST_SHARED_LINK_LIBS})
- endif()
- else()
- set(TEST_LINK_LIBS ${PARQUET_TEST_LINK_LIBS})
- endif()
-
- get_filename_component(TEST_NAME ${REL_TEST_NAME} NAME_WE)
-
- if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${REL_TEST_NAME}.cc)
- # This test has a corresponding .cc file, set it up as an executable.
- set(TEST_PATH "${EXECUTABLE_OUTPUT_PATH}/${TEST_NAME}")
- add_executable(${TEST_NAME} "${REL_TEST_NAME}.cc")
- add_dependencies(unittest ${TEST_NAME})
-
- if(APPLE)
- # On OS X / Thrift >= 0.9.2, tr1/tuple.h is not in libc++
- SET_TARGET_PROPERTIES(${TEST_NAME} PROPERTIES COMPILE_FLAGS
- -DGTEST_USE_OWN_TR1_TUPLE=1)
- else()
- # Linux, for Thrift >= 0.9.2
- SET_TARGET_PROPERTIES(${TEST_NAME} PROPERTIES COMPILE_FLAGS
- -DGTEST_USE_OWN_TR1_TUPLE=0)
- endif()
-
- target_link_libraries(${TEST_NAME} ${TEST_LINK_LIBS})
- else()
- # No executable, just invoke the test (probably a script) directly.
- set(TEST_PATH ${CMAKE_CURRENT_SOURCE_DIR}/${REL_TEST_NAME})
- endif()
-
- if (PARQUET_TEST_MEMCHECK)
- SET_PROPERTY(TARGET ${TEST_NAME}
- APPEND_STRING PROPERTY
- COMPILE_FLAGS " -DPARQUET_VALGRIND")
- add_test(${TEST_NAME}
- valgrind --tool=memcheck --leak-check=full --error-exitcode=1 ${TEST_PATH})
- elseif(MSVC)
- add_test(${TEST_NAME} ${TEST_PATH})
- else()
- add_test(${TEST_NAME}
- ${BUILD_SUPPORT_DIR}/run-test.sh ${CMAKE_BINARY_DIR} test ${TEST_PATH})
- endif()
- set_tests_properties(${TEST_NAME} PROPERTIES LABELS "unittest")
- if(ARGN)
- set_tests_properties(${TEST_NAME} PROPERTIES ${ARGN})
- endif()
-endfunction()
-
-# A wrapper for add_dependencies() that is compatible with PARQUET_BUILD_TESTS.
-function(ADD_PARQUET_TEST_DEPENDENCIES REL_TEST_NAME)
- if(NOT PARQUET_BUILD_TESTS)
- return()
- endif()
- get_filename_component(TEST_NAME ${REL_TEST_NAME} NAME_WE)
-
- add_dependencies(${TEST_NAME} ${ARGN})
-endfunction()
-
-# A wrapper for add_dependencies() that is compatible with PARQUET_BUILD_TESTS.
-function(ADD_PARQUET_LINK_LIBRARIES REL_TEST_NAME)
- if(NOT PARQUET_BUILD_TESTS)
- return()
- endif()
- get_filename_component(TEST_NAME ${REL_TEST_NAME} NAME_WE)
-
- target_link_libraries(${TEST_NAME} ${ARGN})
-endfunction()
-
-enable_testing()
-
-############################################################
-# Dependencies
-############################################################
-
-# Determine compiler version
-include(CompilerInfo)
-include(SetupCxxFlags)
-
-include_directories(${CMAKE_CURRENT_BINARY_DIR}/src)
-include_directories(
- ${CMAKE_CURRENT_SOURCE_DIR}/src
-)
-
-if (PARQUET_MINIMAL_DEPENDENCY)
- set(IGNORE_OPTIONAL_PACKAGES ON)
- message(STATUS "Build using minimal dependencies")
-else()
- set(IGNORE_OPTIONAL_PACKAGES OFF)
-endif()
-include(ThirdpartyToolchain)
-
-set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${CXX_COMMON_FLAGS}")
-set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${PARQUET_CXXFLAGS}")
-
-message(STATUS "CMAKE_CXX_FLAGS: ${CMAKE_CXX_FLAGS}")
-
-# Thrift requires these definitions for some types that we use
-add_definitions(-DHAVE_INTTYPES_H -DHAVE_NETDB_H)
-if (MSVC)
- add_definitions(-DNOMINMAX -D_CRT_SECURE_NO_WARNINGS)
-else()
- add_definitions(-DHAVE_NETINET_IN_H -fPIC)
-endif()
-
-############################################################
-# "make lint" target
-############################################################
-if (UNIX)
- find_program(CPPLINT_BIN NAMES cpplint cpplint.py HINTS ${BUILD_SUPPORT_DIR})
- message(STATUS "Found cpplint executable at ${CPPLINT_BIN}")
- # Full lint
- add_custom_target(lint ${CPPLINT_BIN}
- --verbose=2
- --linelength=90
- --filter=-whitespace/comments,-readability/todo,-build/header_guard,-runtime/references,-readability/check,-build/c++11,-build/include_order
- `find ${CMAKE_CURRENT_SOURCE_DIR}/src ${CMAKE_CURRENT_SOURCE_DIR}/tools ${CMAKE_CURRENT_SOURCE_DIR}/examples ${CMAKE_CURRENT_SOURCE_DIR}/benchmarks -name \\*.cc -or -name \\*.h | sed -e '/parquet\\/parquet_/g'`)
-endif (UNIX)
-
-############################################################
-# "make format" and "make check-format" targets
-############################################################
-
-# runs clang format and updates files in place.
-add_custom_target(format-example
- COMMAND
- ${BUILD_SUPPORT_DIR}/run_clang_format.py
- ${CLANG_FORMAT_BIN}
- ${BUILD_SUPPORT_DIR}/clang_format_exclusions.txt
- ${CMAKE_CURRENT_SOURCE_DIR}/examples)
-
-add_custom_target(format
- DEPENDS format-example
- COMMAND
- ${BUILD_SUPPORT_DIR}/run_clang_format.py
- ${CLANG_FORMAT_BIN}
- ${BUILD_SUPPORT_DIR}/clang_format_exclusions.txt
- ${CMAKE_CURRENT_SOURCE_DIR}/src)
-
-# runs clang format and exits with a non-zero exit code if any files need to be reformatted
-
-# TODO(wesm): Make this work in run_clang_format.py
-add_custom_target(check-format-examples ${BUILD_SUPPORT_DIR}/run_clang_format.py
- ${CLANG_FORMAT_BIN}
- ${BUILD_SUPPORT_DIR}/clang_format_exclusions.txt
- ${CMAKE_CURRENT_SOURCE_DIR}/examples 1)
-add_custom_target(check-format
- DEPENDS check-format-examples
- COMMAND
- ${BUILD_SUPPORT_DIR}/run_clang_format.py
- ${CLANG_FORMAT_BIN}
- ${BUILD_SUPPORT_DIR}/clang_format_exclusions.txt
- ${CMAKE_CURRENT_SOURCE_DIR}/src 1)
-
-############################################################
-# "make clang-tidy" and "make check-clang-tidy" targets
-############################################################
-
-if (${CLANG_TIDY_FOUND})
- # runs clang-tidy and attempts to fix any warning automatically
- add_custom_target(clang-tidy ${BUILD_SUPPORT_DIR}/run-clang-tidy.sh ${CLANG_TIDY_BIN} ${CMAKE_BINARY_DIR}/compile_commands.json 1
- `find ${CMAKE_CURRENT_SOURCE_DIR}/src ${CMAKE_CURRENT_SOURCE_DIR}/tools ${CMAKE_CURRENT_SOURCE_DIR}/examples ${CMAKE_CURRENT_SOURCE_DIR}/benchmarks -name \\*.cc | sed -e '/_types/g' | sed -e '/_constants/g'`)
- # runs clang-tidy and exits with a non-zero exit code if any errors are found.
- add_custom_target(check-clang-tidy ${BUILD_SUPPORT_DIR}/run-clang-tidy.sh ${CLANG_TIDY_BIN} ${CMAKE_BINARY_DIR}/compile_commands.json
- 0 `find ${CMAKE_CURRENT_SOURCE_DIR}/src ${CMAKE_CURRENT_SOURCE_DIR}/tools ${CMAKE_CURRENT_SOURCE_DIR}/examples ${CMAKE_CURRENT_SOURCE_DIR}/benchmarks -name \\*.cc |grep -v -F -f ${CMAKE_CURRENT_SOURCE_DIR}/.clang-tidy-ignore`)
-
-endif()
-
-#############################################################
-# Code coverage
-
-# Adapted from Apache Kudu (incubating)
-if ("${PARQUET_GENERATE_COVERAGE}")
- if("${CMAKE_CXX_COMPILER}" MATCHES ".*clang.*")
- # There appears to be some bugs in clang 3.3 which cause code coverage
- # to have link errors, not locating the llvm_gcda_* symbols.
- # This should be fixed in llvm 3.4 with http://llvm.org/viewvc/llvm-project?view=revision&revision=184666
- message(SEND_ERROR "Cannot currently generate coverage with clang")
- endif()
- message(STATUS "Configuring build for gcov")
- set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} --coverage")
- # For coverage to work properly, we need to use static linkage. Otherwise,
- # __gcov_flush() doesn't properly flush coverage from every module.
- # See http://stackoverflow.com/questions/28164543/using-gcov-flush-within-a-library-doesnt-force-the-other-modules-to-yield-gc
- if(NOT PARQUET_BUILD_STATIC)
- message(SEND_ERROR "Coverage requires the static lib to be built")
- endif()
-endif()
-
-#############################################################
-# Boost linkage
-
-if (PARQUET_BOOST_USE_SHARED)
- set(BOOST_LINK_LIBS
- boost_shared_regex)
- if(MSVC)
- set(BOOST_LINK_LIBS ${BOOST_LINK_LIBS}
- boost_shared_system)
- endif()
-else()
- set(BOOST_LINK_LIBS
- boost_static_regex)
- if(MSVC)
- set(BOOST_LINK_LIBS ${BOOST_LINK_LIBS}
- boost_static_system boost_static_filesystem)
- endif()
-endif()
-
-#############################################################
-# Apache Arrow linkage
-
-if ("${PARQUET_ARROW_LINKAGE}" STREQUAL "shared")
- set(ARROW_LINK_LIBS
- arrow)
-else()
- #############################################################
- # Transitive Library Linkage
-
- if (NOT DEFINED ENV{BROTLI_STATIC_LIB_ENC} OR
- NOT DEFINED ENV{BROTLI_STATIC_LIB_DEC} OR
- NOT DEFINED ENV{BROTLI_STATIC_LIB_COMMON} OR
- NOT DEFINED ENV{SNAPPY_STATIC_LIB} OR
- NOT DEFINED ENV{ZLIB_STATIC_LIB} OR
- NOT DEFINED ENV{LZ4_STATIC_LIB} OR
- NOT DEFINED ENV{ZSTD_STATIC_LIB})
- message(FATAL_ERROR "Missing transitive dependencies for Arrow static linking")
- endif()
-
- set(BROTLI_STATIC_LIB_ENC "$ENV{BROTLI_STATIC_LIB_ENC}")
- set(BROTLI_STATIC_LIB_DEC "$ENV{BROTLI_STATIC_LIB_DEC}")
- set(BROTLI_STATIC_LIB_COMMON "$ENV{BROTLI_STATIC_LIB_COMMON}")
- set(SNAPPY_STATIC_LIB "$ENV{SNAPPY_STATIC_LIB}")
- set(ZLIB_STATIC_LIB "$ENV{ZLIB_STATIC_LIB}")
- set(LZ4_STATIC_LIB "$ENV{LZ4_STATIC_LIB}")
- set(ZSTD_STATIC_LIB "$ENV{ZSTD_STATIC_LIB}")
-
- add_library(brotli_enc STATIC IMPORTED)
- set_target_properties(brotli_enc PROPERTIES IMPORTED_LOCATION ${BROTLI_STATIC_LIB_ENC})
- add_library(brotli_dec STATIC IMPORTED)
- set_target_properties(brotli_dec PROPERTIES IMPORTED_LOCATION ${BROTLI_STATIC_LIB_DEC})
- add_library(brotli_common STATIC IMPORTED)
- set_target_properties(brotli_common PROPERTIES IMPORTED_LOCATION ${BROTLI_STATIC_LIB_COMMON})
- add_library(snappy STATIC IMPORTED)
- set_target_properties(snappy PROPERTIES IMPORTED_LOCATION ${SNAPPY_STATIC_LIB})
- add_library(zlib STATIC IMPORTED)
- set_target_properties(zlib PROPERTIES IMPORTED_LOCATION ${ZLIB_STATIC_LIB})
- add_library(lz4 STATIC IMPORTED)
- set_target_properties(lz4 PROPERTIES IMPORTED_LOCATION ${LZ4_STATIC_LIB})
- add_library(zstd STATIC IMPORTED)
- set_target_properties(zstd PROPERTIES IMPORTED_LOCATION ${ZSTD_STATIC_LIB})
-
- set(TRANSITIVE_LINK_LIBS
- snappy
- zlib
- brotli_enc
- brotli_dec
- brotli_common
- lz4
- zstd
- )
-
- add_definitions(-DARROW_STATIC)
-
- set(ARROW_LINK_LIBS
- arrow_static
- ${TRANSITIVE_LINK_LIBS})
-endif()
-
-#############################################################
-# Test linking
-
-set(PARQUET_MIN_TEST_LIBS
- gtest
- gtest_main)
-
-if (APPLE)
- set(PARQUET_MIN_TEST_LIBS
- ${PARQUET_MIN_TEST_LIBS}
- ${CMAKE_DL_LIBS})
-elseif(NOT MSVC)
- set(PARQUET_MIN_TEST_LIBS
- ${PARQUET_MIN_TEST_LIBS}
- pthread
- ${CMAKE_DL_LIBS})
-endif()
-
-set(PARQUET_TEST_LINK_LIBS ${PARQUET_MIN_TEST_LIBS}
- ${ARROW_LINK_LIBS}
- parquet_static)
-
-set(PARQUET_TEST_SHARED_LINK_LIBS ${PARQUET_MIN_TEST_LIBS}
- parquet_shared)
-
-#############################################################
-# Benchmark linking
-
-if (PARQUET_BUILD_STATIC)
- set(PARQUET_BENCHMARK_LINK_LIBS
- parquet_benchmark_main
- parquet_static)
-else()
- set(PARQUET_BENCHMARK_LINK_LIBS
- parquet_benchmark_main
- parquet_shared)
-endif()
-
-############################################################
-# Generated Thrift sources
-
-set(THRIFT_SRCS
- src/parquet/parquet_constants.cpp
- src/parquet/parquet_types.cpp)
-
-if (NOT MSVC)
- set_source_files_properties(src/parquet/parquet_types.cpp PROPERTIES
- COMPILE_FLAGS -Wno-unused-variable)
-endif()
-
-# List of thrift output targets
-set(THRIFT_OUTPUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/src/parquet)
-set(THRIFT_OUTPUT_FILES "${THRIFT_OUTPUT_DIR}/parquet_types.cpp")
-set(THRIFT_OUTPUT_FILES ${THRIFT_OUTPUT_FILES} "${THRIFT_OUTPUT_DIR}/parquet_types.h")
-set(THRIFT_OUTPUT_FILES ${THRIFT_OUTPUT_FILES} "${THRIFT_OUTPUT_DIR}/parquet_constants.cpp")
-set(THRIFT_OUTPUT_FILES ${THRIFT_OUTPUT_FILES} "${THRIFT_OUTPUT_DIR}/parquet_constants.h")
-
-set_source_files_properties(${THRIFT_OUTPUT_FILES} PROPERTIES GENERATED TRUE)
-
-get_filename_component(ABS_PARQUET_THRIFT src/parquet/parquet.thrift ABSOLUTE)
-
-add_custom_command(
- OUTPUT ${THRIFT_OUTPUT_FILES}
- COMMAND ${THRIFT_COMPILER} --gen cpp -out ${THRIFT_OUTPUT_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/src/parquet/parquet.thrift
- DEPENDS ${ABS_PARQUET_THRIFT} thriftstatic
- COMMENT "Running thrift compiler on parquet.thrift"
- VERBATIM
-)
-
-############################################################
-# Library config
-
-set(LIBPARQUET_SRCS
- src/parquet/arrow/reader.cc
- src/parquet/arrow/record_reader.cc
- src/parquet/arrow/schema.cc
- src/parquet/arrow/writer.cc
- src/parquet/bloom_filter.cc
- src/parquet/column_reader.cc
- src/parquet/column_scanner.cc
- src/parquet/column_writer.cc
- src/parquet/exception.cc
- src/parquet/file_reader.cc
- src/parquet/file_writer.cc
- src/parquet/metadata.cc
- src/parquet/murmur3.cc
- src/parquet/parquet_constants.cpp
- src/parquet/parquet_types.cpp
- src/parquet/printer.cc
- src/parquet/schema.cc
- src/parquet/statistics.cc
- src/parquet/types.cc
- src/parquet/util/comparison.cc
- src/parquet/util/memory.cc
-)
-
-# # Ensure that thrift compilation is done before using its generated headers
-# # in parquet code.
-add_custom_target(thrift-deps ALL
- DEPENDS ${THRIFT_OUTPUT_FILES})
-set(PARQUET_DEPENDENCIES ${PARQUET_DEPENDENCIES} thrift-deps)
-
-if (NOT PARQUET_MINIMAL_DEPENDENCY)
-# These are libraries that we will link privately with parquet_shared (as they
-# do not need to be linked transitively by other linkers), but publicly with
-# parquet_static (because internal users need to transitively link all
-# dependencies)
- set(LIBPARQUET_INTERFACE_LINK_LIBS
- ${ARROW_LINK_LIBS}
- ${BOOST_LINK_LIBS}
- thriftstatic
- )
-# Although we don't link parquet_objlib against anything, we need it to depend
-# on these libs as we may generate their headers via ExternalProject_Add
- set(PARQUET_DEPENDENCIES ${PARQUET_DEPENDENCIES} ${LIBPARQUET_INTERFACE_LINK_LIBS})
-endif()
-
-if(NOT APPLE AND NOT MSVC)
- # Localize thirdparty symbols using a linker version script. This hides them
- # from the client application. The OS X linker does not support the
- # version-script option.
- set(SHARED_LINK_FLAGS "-Wl,--version-script=${CMAKE_CURRENT_SOURCE_DIR}/src/parquet/symbols.map")
-endif()
-
-ADD_LIB(parquet
- SOURCES ${LIBPARQUET_SRCS}
- LIB_BUILD_SHARED ${PARQUET_BUILD_SHARED}
- LIB_BUILD_STATIC ${PARQUET_BUILD_STATIC}
- DEPENDENCIES ${PARQUET_DEPENDENCIES}
- SHARED_LINK_FLAGS ${SHARED_LINK_FLAGS}
- SHARED_PRIVATE_LINK_LIBS ${LIBPARQUET_INTERFACE_LINK_LIBS}
- STATIC_LINK_LIBS ${LIBPARQUET_INTERFACE_LINK_LIBS}
- ABI_VERSION ${PARQUET_ABI_VERSION}
- SO_VERSION ${PARQUET_SO_VERSION}
-)
-
-############################################################
-# Visibility
-############################################################
-# For generate_export_header() and add_compiler_export_flags().
-include(GenerateExportHeader)
-
-# Adapted from Apache Kudu: https://github.com/apache/kudu/commit/bd549e13743a51013585
-# Honor visibility properties for all target types. See
-# "cmake --help-policy CMP0063" for details.
-#
-# This policy was only added to cmake in version 3.3, so until the cmake in
-# thirdparty is updated, we must check if the policy exists before setting it.
-if(POLICY CMP0063)
- cmake_policy(SET CMP0063 NEW)
-endif()
-
-if (PARQUET_BUILD_SHARED)
- if (POLICY CMP0063)
- set_target_properties(parquet_shared
- PROPERTIES C_VISIBILITY_PRESET hidden)
- set_target_properties(parquet_shared
- PROPERTIES CXX_VISIBILITY_PRESET hidden)
- set_target_properties(parquet_shared
- PROPERTIES VISIBILITY_INLINES_HIDDEN 1)
- else()
- # Sets -fvisibility=hidden for gcc
- add_compiler_export_flags()
- endif()
-endif()
-
-add_subdirectory(src/parquet)
-add_subdirectory(src/parquet/api)
-add_subdirectory(src/parquet/arrow)
-add_subdirectory(src/parquet/util)
-
-if (NOT MSVC)
- add_subdirectory(benchmarks)
-endif()
-add_subdirectory(examples/low-level-api)
-add_subdirectory(tools)
-
-add_custom_target(clean-all
- COMMAND ${CMAKE_BUILD_TOOL} clean
- COMMAND ${CMAKE_COMMAND} -P cmake_modules/clean-all.cmake
-)
diff --git a/NOTICE.txt b/NOTICE.txt
deleted file mode 100644
index a1ab98e..0000000
--- a/NOTICE.txt
+++ /dev/null
@@ -1,5 +0,0 @@
-Apache Parquet
-Copyright 2016 The Apache Software Foundation
-
-This product includes software developed at
-The Apache Software Foundation (http://www.apache.org/).
diff --git a/README.md b/README.md
index a42e7b4..cf0c62f 100644
--- a/README.md
+++ b/README.md
@@ -12,271 +12,11 @@
limitations under the License. See accompanying LICENSE file.
-->
-## Note: Development for Apache Parquet in C++ has moved
+# Note: Development for Apache Parquet in C++ has moved
-The Apache Arrow and Parquet have merged development process and build systems
-in the Arrow repository. Please submit pull requests in
+The Apache Arrow and Parquet C++ projects have merged development process and
+build systems in the Arrow repository. Please submit pull requests in
https://github.com/apache/arrow.
JIRA issues should continue to be opened in the PARQUET JIRA project.
-
-## Apache Parquet for C++: a C++ library to read and write the Apache Parquet
- columnar data format.
-
-<table>
- <tr>
- <td>Build Status</td>
- <td>
- <a href="https://travis-ci.org/apache/parquet-cpp">
- <img src="https://travis-ci.org/apache/parquet-cpp.svg?branch=master" alt="travis build status" />
- </a>
- </td>
- </tr>
- <tr>
- <td>Test coverage</td>
- <td>
- <a href='https://coveralls.io/github/apache/parquet-cpp?branch=master'><img src='https://coveralls.io/repos/github/apache/parquet-cpp/badge.svg?branch=master' alt='Coverage Status' /></a>
- </td>
- </tr>
-</table>
-
-## System Dependencies
-
-We use the CMake build system and require a minimum version of 3.2. If you are
-using an older Linux distribution, you may need to use a PPA (for apt users) or
-build CMake from source.
-
-### Linux
-
-parquet-cpp requires gcc 4.8 or higher on Linux.
-
-To build parquet-cpp out of the box, you must install some build prerequisites
-for the thirdparty dependencies. On Debian/Ubuntu, these can be installed with:
-
-```
-sudo apt-get install libboost-dev libboost-filesystem-dev \
- libboost-program-options-dev libboost-regex-dev \
- libboost-system-dev libboost-test-dev \
- libssl-dev libtool bison flex pkg-config
-```
-
-### OS X
-
-You must use XCode 6 or higher. We recommend using Homebrew to install Boost,
-which is required for Thrift:
-
-```
-brew install boost
-```
-
-### Windows
-
-Check [Windows developer guide][1] for instructions to build parquet-cpp on Windows.
-
-## Third Party Dependencies
-
-- Apache Arrow >= 0.7.0 (memory management, compression, IO, optional columnar
- data adapters)
-- Thrift 0.7+ [install instructions](https://thrift.apache.org/docs/install/)
-- googletest 1.7.0 (cannot be installed with package managers)
-- Google Benchmark (only required if building benchmarks)
-
-You can either install these dependencies separately, otherwise they will be
-built automatically as part of the build.
-
-Symbols from Thrift, Snappy, and ZLib are statically-linked into the
-`libparquet` shared library, so these dependencies must be built with `-fPIC`
-on Linux and OS X. Since Linux package managers do not consistently compile the
-static libraries for these components with `-fPIC`, you may have issues with
-Linux packages such as `libsnappy-dev`. It may be easier to depend on the
-thirdparty toolchain that parquet-cpp builds automatically.
-
-## Build
-
-- `cmake .`
-
- - You can customize build dependency locations through various environment variables:
- - ARROW_HOME customizes the Apache Arrow installed location.
- - THRIFT_HOME customizes the Apache Thrift (C++ libraries and compiler
- installed location.
- - GTEST_HOME customizes the googletest installed location (if you are
- building the unit tests).
- - GBENCHMARK_HOME customizes the Google Benchmark installed location (if
- you are building the benchmarks).
-
-- `make`
-
-The binaries will be built to ./debug which contains the libraries to link against as
-well as a few example executables.
-
-To disable the testing (which requires `googletest`), pass
-`-DPARQUET_BUILD_TESTS=Off` to `cmake`.
-
-For release-level builds (enable optimizations and disable debugging), pass
-`-DCMAKE_BUILD_TYPE=Release` to `cmake`.
-
-To build only the library with minimal dependencies, pass
-`-DPARQUET_MINIMAL_DEPENDENCY=ON` to `cmake`.
-Note that the executables, tests, and benchmarks should be disabled as well.
-
-Incremental builds can be done afterwords with just `make`.
-
-## Using with Apache Arrow
-
-Arrow provides some of the memory management and IO interfaces that we use in
-parquet-cpp. By default, Parquet links to Arrow's shared libraries. If you wish
-to statically-link the Arrow symbols instead, pass
-`-DPARQUET_ARROW_LINKAGE=static`.
-
-## Testing
-
-This library uses Google's `googletest` unit test framework. After building
-with `make`, you can run the test suite by running
-
-```
-make unittest
-```
-
-The test suite relies on an environment variable `PARQUET_TEST_DATA` pointing
-to the `data` directory in the source checkout, for example:
-
-```
-export PARQUET_TEST_DATA=`pwd`/data
-```
-
-See `ctest --help` for configuration details about ctest. On GNU/Linux systems,
-you can use valgrind with ctest to look for memory leaks:
-
-```
-valgrind --tool=memcheck --leak-check=yes ctest
-```
-
-## Building/Running benchmarks
-
-Follow the directions for simple build except run cmake
-with the `--PARQUET_BUILD_BENCHMARKS` parameter set correctly:
-
- cmake -DPARQUET_BUILD_BENCHMARKS=ON ..
-
-and instead of make unittest run either `make; ctest` to run both unit tests
-and benchmarks or `make runbenchmark` to run only the benchmark tests.
-
-Benchmark logs will be placed in the build directory under `build/benchmark-logs`.
-
-
-## Out-of-source builds
-
-parquet-cpp supports out of source builds. For example:
-
-```
-mkdir test-build
-cd test-build
-cmake ..
-make
-ctest -L unittest
-```
-
-By using out-of-source builds you can preserve your current build state in case
-you need to switch to another git branch.
-
-Design
-========
-The library consists of 3 layers that map to the 3 units in the parquet format.
-
-The first is the encodings which correspond to data pages. The APIs at this level
-return single values.
-
-The second layer is the column reader which corresponds to column chunks. The APIs at
-this level return a triple: definition level, repetition level and value. It also handles
-reading pages, compression and managing encodings.
-
-The 3rd layer would handle reading/writing records.
-
-Developer Notes
-========
-The project adheres to the google coding convention:
-http://google-styleguide.googlecode.com/svn/trunk/cppguide.xml
-with two notable exceptions. We do not encourage anonymous namespaces and the line
-length is 90 characters.
-
-You can run `cpplint` through the build system with
-
-```
-make lint
-```
-
-The project prefers the use of C++ style memory management. new/delete should be used
-over malloc/free. new/delete should be avoided whenever possible by using stl/boost
-where possible. For example, scoped_ptr instead of explicit new/delete and using
-std::vector instead of allocated buffers. Currently, c++11 features are not used.
-
-For error handling, this project uses exceptions.
-
-In general, many of the APIs at the layers are interface based for extensibility. To
-minimize the cost of virtual calls, the APIs should be batch-centric. For example,
-encoding should operate on batches of values rather than a single value.
-
-## Using clang with a custom gcc toolchain
-
-Suppose you are building libraries with a thirdparty gcc toolchain (not a
-built-in system one) on Linux. To use clang for development while linking to
-the proper toolchain, you can do (for out of source builds):
-
-```shell
-export CMAKE_CLANG_OPTIONS=--gcc-toolchain=$TOOLCHAIN/gcc-4.9.2
-
-export CC=$TOOLCHAIN/llvm-3.7.0/bin/clang
-export CXX=$TOOLCHAIN/llvm-3.7.0/bin/clang++
-
-cmake -DCMAKE_CLANG_OPTIONS=$CMAKE_CLANG_OPTIONS \
- -DCMAKE_CXX_FLAGS="-Werror" ..
-```
-
-## Code Coverage
-
-To build with `gcov` code coverage and upload results to http://coveralls.io or
-http://codecov.io, here are some instructions.
-
-First, build the project with coverage and run the test suite
-
-```
-cd $PARQUET_HOME
-mkdir coverage-build
-cd coverage-build
-cmake -DPARQUET_GENERATE_COVERAGE=1
-make -j$PARALLEL
-ctest -L unittest
-```
-
-The `gcov` artifacts are not located in a place that works well with either
-coveralls or codecov, so there is a helper script you need to run
-
-```
-mkdir coverage_artifacts
-python ../build-support/collect_coverage.py CMakeFiles/parquet.dir/src/ coverage_artifacts
-```
-
-For codecov.io (using the provided project token -- be sure to keep this
-private):
-
-```
-cd coverage_artifacts
-codecov --token $PARQUET_CPP_CODECOV_TOKEN --gcov-args '\-l' --root $PARQUET_ROOT
-```
-
-For coveralls, install `cpp_coveralls`:
-
-```
-pip install cpp_coveralls
-```
-
-And the coveralls upload script:
-
-```
-coveralls -t $PARQUET_CPP_COVERAGE_TOKEN --gcov-options '\-l' -r $PARQUET_ROOT --exclude $PARQUET_ROOT/thirdparty --exclude $PARQUET_ROOT/build --exclude $NATIVE_TOOLCHAIN --exclude $PARQUET_ROOT/src/parquet/thrift
-```
-
-Note that `gcov` throws off artifacts from the STL, so I excluded my toolchain
-root stored in `$NATIVE_TOOLCHAIN` to avoid a cluttered coverage report.
-
-[1]: https://github.com/apache/parquet-cpp/blob/master/docs/Windows.md
+Discussions continue to be help on the Parquet dev mailing list.
diff --git a/appveyor.yml b/appveyor.yml
deleted file mode 100644
index 5cc8354..0000000
--- a/appveyor.yml
+++ /dev/null
@@ -1,68 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-# Operating system (build VM template)
-os: Visual Studio 2015
-
-matrix:
- fast_finish: true
-
-environment:
- matrix:
- - GENERATOR: NMake Makefiles
- PYTHON: "3.5"
- ARCH: "64"
- CONFIGURATION: "Release"
- - GENERATOR: Visual Studio 14 2015 Win64
- PYTHON: "3.5"
- ARCH: "64"
- CONFIGURATION: "Debug"
- - GENERATOR: Visual Studio 15 2017 Win64
- PYTHON: "3.5"
- ARCH: "64"
- CONFIGURATION: "Release"
- APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017
- BOOST_ROOT: C:\Libraries\boost_1_64_0
- - GENERATOR: Visual Studio 14 2015 Win64
- PYTHON: "3.5"
- ARCH: "64"
- CONFIGURATION: "Debug"
- USE_STATIC_CRT: "ON"
- - GENERATOR: Visual Studio 14 2015 Win64
- PYTHON: "3.5"
- ARCH: "64"
- CONFIGURATION: "Release"
- USE_STATIC_CRT: "ON"
- - GENERATOR: Visual Studio 14 2015 Win64
- PYTHON: "3.5"
- ARCH: "64"
- CONFIGURATION: "Toolchain"
-
- MSVC_DEFAULT_OPTIONS: ON
- BOOST_ROOT: C:\Libraries\boost_1_63_0
- BOOST_LIBRARYDIR: C:\Libraries\boost_1_63_0\lib64-msvc-14.0
- USE_CLCACHE: false
-
-init:
- - set MINICONDA=C:\Miniconda35-x64
- - set PATH=%MINICONDA%;%MINICONDA%/Scripts;%MINICONDA%/Library/bin;%PATH%
-
-build_script:
- - call ci\msvc-build.bat
-
-# Disable test discovery
-test: off
diff --git a/benchmarks/CMakeLists.txt b/benchmarks/CMakeLists.txt
deleted file mode 100644
index 9566aec..0000000
--- a/benchmarks/CMakeLists.txt
+++ /dev/null
@@ -1,32 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-SET(LINK_LIBS
- ${ARROW_LINK_LIBS}
- ${PARQUET_MIN_TEST_LIBS}
- thriftstatic)
-
-if (PARQUET_BUILD_BENCHMARKS)
- add_executable(decode_benchmark decode_benchmark.cc)
-
- add_dependencies(decode_benchmark gtest)
-
- # This uses private APIs
- target_link_libraries(decode_benchmark
- ${LINK_LIBS}
- parquet_static)
-endif()
diff --git a/benchmarks/decode_benchmark.cc b/benchmarks/decode_benchmark.cc
deleted file mode 100644
index 3ae32b4..0000000
--- a/benchmarks/decode_benchmark.cc
+++ /dev/null
@@ -1,471 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#include <stdio.h>
-#include <iostream>
-#include <random>
-
-#include "arrow/test-util.h"
-#include "arrow/util/compression.h"
-#include "arrow/util/compression_snappy.h"
-
-#include "parquet/encoding-internal.h"
-#include "parquet/util/logging.h"
-#include "parquet/util/stopwatch.h"
-
-/**
- * Test bed for encodings and some utilities to measure their throughput.
- * TODO: this file needs some major cleanup.
- */
-
-class DeltaBitPackEncoder {
- public:
- explicit DeltaBitPackEncoder(int mini_block_size = 8) {
- mini_block_size_ = mini_block_size;
- }
-
- void Add(int64_t v) { values_.push_back(v); }
-
- uint8_t* Encode(int* encoded_len) {
- uint8_t* result = new uint8_t[10 * 1024 * 1024];
- int num_mini_blocks = static_cast<int>(arrow::BitUtil::CeilDiv(num_values() - 1,
- mini_block_size_));
- uint8_t* mini_block_widths = NULL;
-
- arrow::BitWriter writer(result, 10 * 1024 * 1024);
-
- // Writer the size of each block. We only use 1 block currently.
- writer.PutVlqInt(static_cast<uint32_t>(num_mini_blocks * mini_block_size_));
-
- // Write the number of mini blocks.
- writer.PutVlqInt(static_cast<uint32_t>(num_mini_blocks));
-
- // Write the number of values.
- writer.PutVlqInt(num_values() - 1);
-
- // Write the first value.
- writer.PutZigZagVlqInt(static_cast<uint32_t>(values_[0]));
-
- // Compute the values as deltas and the min delta.
- int64_t min_delta = std::numeric_limits<int64_t>::max();
- for (size_t i = values_.size() - 1; i > 0; --i) {
- values_[i] -= values_[i - 1];
- min_delta = std::min(min_delta, values_[i]);
- }
-
- // Write out the min delta.
- writer.PutZigZagVlqInt(static_cast<int32_t>(min_delta));
-
- // We need to save num_mini_blocks bytes to store the bit widths of the mini
- // blocks.
- mini_block_widths = writer.GetNextBytePtr(num_mini_blocks);
-
- int idx = 1;
- for (int i = 0; i < num_mini_blocks; ++i) {
- int n = std::min(mini_block_size_, num_values() - idx);
-
- // Compute the max delta in this mini block.
- int64_t max_delta = std::numeric_limits<int64_t>::min();
- for (int j = 0; j < n; ++j) {
- max_delta = std::max(values_[idx + j], max_delta);
- }
-
- // The bit width for this block is the number of bits needed to store
- // (max_delta - min_delta).
- int bit_width = arrow::BitUtil::NumRequiredBits(max_delta - min_delta);
- mini_block_widths[i] = static_cast<uint8_t>(bit_width);
-
- // Encode this mini blocking using min_delta and bit_width
- for (int j = 0; j < n; ++j) {
- writer.PutValue(values_[idx + j] - min_delta, bit_width);
- }
-
- // Pad out the last block.
- for (int j = n; j < mini_block_size_; ++j) {
- writer.PutValue(0, bit_width);
- }
- idx += n;
- }
-
- writer.Flush();
- *encoded_len = writer.bytes_written();
- return result;
- }
-
- int num_values() const { return static_cast<int>(values_.size()); }
-
- private:
- int mini_block_size_;
- std::vector<int64_t> values_;
-};
-
-class DeltaLengthByteArrayEncoder {
- public:
- explicit DeltaLengthByteArrayEncoder(int mini_block_size = 8)
- : len_encoder_(mini_block_size),
- buffer_(new uint8_t[10 * 1024 * 1024]),
- offset_(0),
- plain_encoded_len_(0) {}
-
- void Add(const std::string& s) {
- Add(reinterpret_cast<const uint8_t*>(s.data()), static_cast<int>(s.size()));
- }
-
- void Add(const uint8_t* ptr, int len) {
- plain_encoded_len_ += static_cast<int>(len + sizeof(int));
- len_encoder_.Add(len);
- memcpy(buffer_ + offset_, ptr, len);
- offset_ += len;
- }
-
- uint8_t* Encode(int* encoded_len) {
- uint8_t* encoded_lengths = len_encoder_.Encode(encoded_len);
- memmove(buffer_ + *encoded_len + sizeof(int), buffer_, offset_);
- memcpy(buffer_, encoded_len, sizeof(int));
- memcpy(buffer_ + sizeof(int), encoded_lengths, *encoded_len);
- *encoded_len += static_cast<int>(offset_ + sizeof(int));
- return buffer_;
- }
-
- int num_values() const { return len_encoder_.num_values(); }
- int plain_encoded_len() const { return plain_encoded_len_; }
-
- private:
- DeltaBitPackEncoder len_encoder_;
- uint8_t* buffer_;
- int offset_;
- int plain_encoded_len_;
-};
-
-class DeltaByteArrayEncoder {
- public:
- DeltaByteArrayEncoder() : plain_encoded_len_(0) {}
-
- void Add(const std::string& s) {
- plain_encoded_len_ += static_cast<int>(s.size() + sizeof(int));
- int min_len = static_cast<int>(std::min(s.size(), last_value_.size()));
- int prefix_len = 0;
- for (int i = 0; i < min_len; ++i) {
- if (s[i] == last_value_[i]) {
- ++prefix_len;
- } else {
- break;
- }
- }
- prefix_len_encoder_.Add(prefix_len);
- suffix_encoder_.Add(reinterpret_cast<const uint8_t*>(s.data()) + prefix_len,
- static_cast<int>(s.size() - prefix_len));
- last_value_ = s;
- }
-
- uint8_t* Encode(int* encoded_len) {
- int prefix_buffer_len;
- uint8_t* prefix_buffer = prefix_len_encoder_.Encode(&prefix_buffer_len);
- int suffix_buffer_len;
- uint8_t* suffix_buffer = suffix_encoder_.Encode(&suffix_buffer_len);
-
- uint8_t* buffer = new uint8_t[10 * 1024 * 1024];
- memcpy(buffer, &prefix_buffer_len, sizeof(int));
- memcpy(buffer + sizeof(int), prefix_buffer, prefix_buffer_len);
- memcpy(buffer + sizeof(int) + prefix_buffer_len, suffix_buffer, suffix_buffer_len);
- *encoded_len = static_cast<int>(sizeof(int) + prefix_buffer_len + suffix_buffer_len);
- return buffer;
- }
-
- int num_values() const { return prefix_len_encoder_.num_values(); }
- int plain_encoded_len() const { return plain_encoded_len_; }
-
- private:
- DeltaBitPackEncoder prefix_len_encoder_;
- DeltaLengthByteArrayEncoder suffix_encoder_;
- std::string last_value_;
- int plain_encoded_len_;
-};
-
-uint64_t TestPlainIntEncoding(const uint8_t* data, int num_values, int batch_size) {
- uint64_t result = 0;
- parquet::PlainDecoder<parquet::Int64Type> decoder(nullptr);
- decoder.SetData(num_values, data, static_cast<int>(num_values * sizeof(int64_t)));
- std::vector<int64_t> values(batch_size);
- for (int i = 0; i < num_values;) {
- int n = decoder.Decode(values.data(), batch_size);
- for (int j = 0; j < n; ++j) {
- result += values[j];
- }
- i += n;
- }
- return result;
-}
-
-uint64_t TestBinaryPackedEncoding(const char* name, const std::vector<int64_t>& values,
- int benchmark_iters = -1,
- int benchmark_batch_size = 1) {
- int mini_block_size;
- if (values.size() < 8) {
- mini_block_size = 8;
- } else if (values.size() < 16) {
- mini_block_size = 16;
- } else {
- mini_block_size = 32;
- }
- parquet::DeltaBitPackDecoder<parquet::Int64Type> decoder(nullptr);
- DeltaBitPackEncoder encoder(mini_block_size);
- for (size_t i = 0; i < values.size(); ++i) {
- encoder.Add(values[i]);
- }
-
- int raw_len = static_cast<int>(encoder.num_values() * sizeof(int));
- int len;
- uint8_t* buffer = encoder.Encode(&len);
-
- if (benchmark_iters == -1) {
- printf("%s\n", name);
- printf(" Raw len: %d\n", raw_len);
- printf(" Encoded len: %d (%0.2f%%)\n", len,
- static_cast<float>(len) * 100.0f / static_cast<float>(raw_len));
- decoder.SetData(encoder.num_values(), buffer, len);
- for (int i = 0; i < encoder.num_values(); ++i) {
- int64_t x = 0;
- decoder.Decode(&x, 1);
- if (values[i] != x) {
- std::cerr << "Bad: " << i << std::endl;
- std::cerr << " " << x << " != " << values[i] << std::endl;
- break;
- }
- }
- return 0;
- } else {
- printf("%s\n", name);
- printf(" Raw len: %d\n", raw_len);
- printf(" Encoded len: %d (%0.2f%%)\n", len,
- static_cast<float>(len) * 100.0f / static_cast<float>(raw_len));
-
- uint64_t result = 0;
- std::vector<int64_t> buf(benchmark_batch_size);
- parquet::StopWatch sw;
- sw.Start();
- for (int k = 0; k < benchmark_iters; ++k) {
- decoder.SetData(encoder.num_values(), buffer, len);
- for (size_t i = 0; i < values.size();) {
- int n = decoder.Decode(buf.data(), benchmark_batch_size);
- for (int j = 0; j < n; ++j) {
- result += buf[j];
- }
- i += n;
- }
- }
- uint64_t elapsed = sw.Stop();
- double num_ints = static_cast<double>(values.size() * benchmark_iters) * 1000.;
- printf("%s rate (batch size = %2d): %0.3fM per second.\n", name, benchmark_batch_size,
- num_ints / static_cast<double>(elapsed));
- return result;
- }
-}
-
-#define DECODE_TEST(NAME, FN, DATA, BATCH_SIZE) \
- sw.Start(); \
- for (int i = 0; i < NUM_ITERS; ++i) { \
- FN(reinterpret_cast<uint8_t*>(&DATA[0]), NUM_VALUES, BATCH_SIZE); \
- } \
- elapsed = sw.Stop(); \
- printf("%s rate (batch size = %2d): %0.3fM per second.\n", NAME, BATCH_SIZE, \
- mult / static_cast<double>(elapsed));
-
-void TestPlainIntCompressed(::arrow::Codec* codec, const std::vector<int64_t>& data,
- int num_iters, int batch_size) {
- const uint8_t* raw_data = reinterpret_cast<const uint8_t*>(&data[0]);
- int uncompressed_len = static_cast<int>(data.size() * sizeof(int64_t));
- uint8_t* decompressed_data = new uint8_t[uncompressed_len];
-
- int64_t max_compressed_size = codec->MaxCompressedLen(uncompressed_len, raw_data);
- uint8_t* compressed_data = new uint8_t[max_compressed_size];
- int64_t compressed_len;
- DCHECK(codec
- ->Compress(uncompressed_len, raw_data, max_compressed_size, compressed_data,
- &compressed_len)
- .ok());
-
- printf("\n%s:\n Uncompressed len: %d\n Compressed len: %d\n", codec->name(),
- uncompressed_len, static_cast<int>(compressed_len));
-
- double mult = static_cast<double>(num_iters * data.size()) * 1000.;
- parquet::StopWatch sw;
- sw.Start();
- uint64_t r = 0;
- for (int i = 0; i < num_iters; ++i) {
- ABORT_NOT_OK(codec->Decompress(compressed_len, compressed_data, uncompressed_len,
- decompressed_data));
- r += TestPlainIntEncoding(decompressed_data, static_cast<int>(data.size()),
- batch_size);
- }
- int64_t elapsed = sw.Stop();
- printf("Compressed(%s) plain int rate (batch size = %2d): %0.3fM per second.\n",
- codec->name(), batch_size, mult / static_cast<double>(elapsed));
-
- delete[] compressed_data;
- delete[] decompressed_data;
-}
-
-void TestBinaryPacking() {
- std::vector<int64_t> values;
- values.clear();
- for (int i = 0; i < 100; ++i) values.push_back(0);
- TestBinaryPackedEncoding("Zeros", values);
-
- values.clear();
- for (int i = 1; i <= 5; ++i) values.push_back(i);
- TestBinaryPackedEncoding("Example 1", values);
-
- values.clear();
- values.push_back(7);
- values.push_back(5);
- values.push_back(3);
- values.push_back(1);
- values.push_back(2);
- values.push_back(3);
- values.push_back(4);
- values.push_back(5);
- TestBinaryPackedEncoding("Example 2", values);
-
- // Test rand ints between 0 and 10K
- values.clear();
- int seed = 0;
- std::mt19937 gen(seed);
- std::uniform_int_distribution<int> d(0, 10000);
- for (int i = 0; i < 500000; ++i) {
- values.push_back(d(gen));
- }
- TestBinaryPackedEncoding("Rand [0, 10000)", values);
-
- // Test rand ints between 0 and 100
- values.clear();
- std::uniform_int_distribution<int> d1(0, 100);
- for (int i = 0; i < 500000; ++i) {
- values.push_back(d1(gen));
- }
- TestBinaryPackedEncoding("Rand [0, 100)", values);
-}
-
-void TestDeltaLengthByteArray() {
- parquet::DeltaLengthByteArrayDecoder decoder(nullptr);
- DeltaLengthByteArrayEncoder encoder;
-
- std::vector<std::string> values;
- values.push_back("Hello");
- values.push_back("World");
- values.push_back("Foobar");
- values.push_back("ABCDEF");
-
- for (size_t i = 0; i < values.size(); ++i) {
- encoder.Add(values[i]);
- }
-
- int len = 0;
- uint8_t* buffer = encoder.Encode(&len);
- printf("DeltaLengthByteArray\n Raw len: %d\n Encoded len: %d\n",
- encoder.plain_encoded_len(), len);
- decoder.SetData(encoder.num_values(), buffer, len);
- for (int i = 0; i < encoder.num_values(); ++i) {
- parquet::ByteArray v = {0, NULL};
- decoder.Decode(&v, 1);
- std::string r = std::string(reinterpret_cast<const char*>(v.ptr), v.len);
- if (r != values[i]) {
- std::cout << "Bad " << r << " != " << values[i] << std::endl;
- }
- }
-}
-
-void TestDeltaByteArray() {
- parquet::DeltaByteArrayDecoder decoder(nullptr);
- DeltaByteArrayEncoder encoder;
-
- std::vector<std::string> values;
-
- // Wikipedia example
- values.push_back("myxa");
- values.push_back("myxophyta");
- values.push_back("myxopod");
- values.push_back("nab");
- values.push_back("nabbed");
- values.push_back("nabbing");
- values.push_back("nabit");
- values.push_back("nabk");
- values.push_back("nabob");
- values.push_back("nacarat");
- values.push_back("nacelle");
-
- for (size_t i = 0; i < values.size(); ++i) {
- encoder.Add(values[i]);
- }
-
- int len = 0;
- uint8_t* buffer = encoder.Encode(&len);
- printf("DeltaLengthByteArray\n Raw len: %d\n Encoded len: %d\n",
- encoder.plain_encoded_len(), len);
- decoder.SetData(encoder.num_values(), buffer, len);
- for (int i = 0; i < encoder.num_values(); ++i) {
- parquet::ByteArray v;
- decoder.Decode(&v, 1);
- std::string r = std::string(reinterpret_cast<const char*>(v.ptr), v.len);
- if (r != values[i]) {
- std::cout << "Bad " << r << " != " << values[i] << std::endl;
- }
- }
-}
-
-int main(int argc, char** argv) {
- TestBinaryPacking();
- TestDeltaLengthByteArray();
- TestDeltaByteArray();
-
- parquet::StopWatch sw;
- uint64_t elapsed = 0;
-
- const int NUM_VALUES = 1024 * 1024;
- const int NUM_ITERS = 10;
- const double mult = NUM_VALUES * NUM_ITERS * 1000.;
-
- std::vector<int64_t> plain_int_data;
- plain_int_data.resize(NUM_VALUES);
-
- DECODE_TEST("Plain decoder", TestPlainIntEncoding, plain_int_data, 1);
- DECODE_TEST("Plain decoder", TestPlainIntEncoding, plain_int_data, 16);
- DECODE_TEST("Plain decoder", TestPlainIntEncoding, plain_int_data, 32);
- DECODE_TEST("Plain decoder", TestPlainIntEncoding, plain_int_data, 64);
-
- // Test rand ints between 0 and 10K
- std::vector<int64_t> values;
- int seed = 0;
- std::mt19937 gen(seed);
- std::uniform_int_distribution<int> d(0, 10000);
- for (int i = 0; i < 1000000; ++i) {
- values.push_back(d(gen));
- }
- TestBinaryPackedEncoding("Rand 0-10K", values, 100, 1);
- TestBinaryPackedEncoding("Rand 0-10K", values, 100, 16);
- TestBinaryPackedEncoding("Rand 0-10K", values, 100, 32);
- TestBinaryPackedEncoding("Rand 0-10K", values, 100, 64);
-
- ::arrow::SnappyCodec snappy_codec;
-
- TestPlainIntCompressed(&snappy_codec, values, 100, 1);
- TestPlainIntCompressed(&snappy_codec, values, 100, 16);
- TestPlainIntCompressed(&snappy_codec, values, 100, 32);
- TestPlainIntCompressed(&snappy_codec, values, 100, 64);
-
- return 0;
-}
diff --git a/build-support/clang_format_exclusions.txt b/build-support/clang_format_exclusions.txt
deleted file mode 100644
index ce72d2f..0000000
--- a/build-support/clang_format_exclusions.txt
+++ /dev/null
@@ -1 +0,0 @@
-*_generated*
diff --git a/build-support/collect_coverage.py b/build-support/collect_coverage.py
deleted file mode 100755
index 6712760..0000000
--- a/build-support/collect_coverage.py
+++ /dev/null
@@ -1,43 +0,0 @@
-#!/usr/bin/env python
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-import os
-import os.path as osp
-import shutil
-import sys
-
-
-def is_coverage_file(path):
- return path.endswith('gcno') or path.endswith('gcda')
-
-
-def copy_files(path, outpath='.'):
- for root, dirs, files in os.walk(path):
- for fname in files:
- if not is_coverage_file(fname):
- continue
- relpath = osp.join(root, fname)
- dstpath = '_'.join((root.replace(path, '').replace('/', '_'),
- fname))
-
- shutil.copy(relpath, osp.join(outpath, dstpath))
-
-if __name__ == '__main__':
- path = sys.argv[1]
- outpath = sys.argv[2]
- copy_files(path, outpath)
diff --git a/build-support/cpplint.py b/build-support/cpplint.py
deleted file mode 100755
index ece520b..0000000
--- a/build-support/cpplint.py
+++ /dev/null
@@ -1,6323 +0,0 @@
-#!/usr/bin/env python
-#
-# Copyright (c) 2009 Google Inc. All rights reserved.
-#
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions are
-# met:
-#
-# * Redistributions of source code must retain the above copyright
-# notice, this list of conditions and the following disclaimer.
-# * Redistributions in binary form must reproduce the above
-# copyright notice, this list of conditions and the following disclaimer
-# in the documentation and/or other materials provided with the
-# distribution.
-# * Neither the name of Google Inc. nor the names of its
-# contributors may be used to endorse or promote products derived from
-# this software without specific prior written permission.
-#
-# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-"""Does google-lint on c++ files.
-
-The goal of this script is to identify places in the code that *may*
-be in non-compliance with google style. It does not attempt to fix
-up these problems -- the point is to educate. It does also not
-attempt to find all problems, or to ensure that everything it does
-find is legitimately a problem.
-
-In particular, we can get very confused by /* and // inside strings!
-We do a small hack, which is to ignore //'s with "'s after them on the
-same line, but it is far from perfect (in either direction).
-"""
-
-import codecs
-import copy
-import getopt
-import math # for log
-import os
-import re
-import sre_compile
-import string
-import sys
-import unicodedata
-
-
-_USAGE = """
-Syntax: cpplint.py [--verbose=#] [--output=vs7] [--filter=-x,+y,...]
- [--counting=total|toplevel|detailed] [--root=subdir]
- [--linelength=digits]
- <file> [file] ...
-
- The style guidelines this tries to follow are those in
- http://google-styleguide.googlecode.com/svn/trunk/cppguide.xml
-
- Every problem is given a confidence score from 1-5, with 5 meaning we are
- certain of the problem, and 1 meaning it could be a legitimate construct.
- This will miss some errors, and is not a substitute for a code review.
-
- To suppress false-positive errors of a certain category, add a
- 'NOLINT(category)' comment to the line. NOLINT or NOLINT(*)
- suppresses errors of all categories on that line.
-
- The files passed in will be linted; at least one file must be provided.
- Default linted extensions are .cc, .cpp, .cu, .cuh and .h. Change the
- extensions with the --extensions flag.
-
- Flags:
-
- output=vs7
- By default, the output is formatted to ease emacs parsing. Visual Studio
- compatible output (vs7) may also be used. Other formats are unsupported.
-
- verbose=#
- Specify a number 0-5 to restrict errors to certain verbosity levels.
-
- filter=-x,+y,...
- Specify a comma-separated list of category-filters to apply: only
- error messages whose category names pass the filters will be printed.
- (Category names are printed with the message and look like
- "[whitespace/indent]".) Filters are evaluated left to right.
- "-FOO" and "FOO" means "do not print categories that start with FOO".
- "+FOO" means "do print categories that start with FOO".
-
- Examples: --filter=-whitespace,+whitespace/braces
- --filter=whitespace,runtime/printf,+runtime/printf_format
- --filter=-,+build/include_what_you_use
-
- To see a list of all the categories used in cpplint, pass no arg:
- --filter=
-
- counting=total|toplevel|detailed
- The total number of errors found is always printed. If
- 'toplevel' is provided, then the count of errors in each of
- the top-level categories like 'build' and 'whitespace' will
- also be printed. If 'detailed' is provided, then a count
- is provided for each category like 'build/class'.
-
- root=subdir
- The root directory used for deriving header guard CPP variable.
- By default, the header guard CPP variable is calculated as the relative
- path to the directory that contains .git, .hg, or .svn. When this flag
- is specified, the relative path is calculated from the specified
- directory. If the specified directory does not exist, this flag is
- ignored.
-
- Examples:
- Assuming that src/.git exists, the header guard CPP variables for
- src/chrome/browser/ui/browser.h are:
-
- No flag => CHROME_BROWSER_UI_BROWSER_H_
- --root=chrome => BROWSER_UI_BROWSER_H_
- --root=chrome/browser => UI_BROWSER_H_
-
- linelength=digits
- This is the allowed line length for the project. The default value is
- 80 characters.
-
- Examples:
- --linelength=120
-
- extensions=extension,extension,...
- The allowed file extensions that cpplint will check
-
- Examples:
- --extensions=hpp,cpp
-
- cpplint.py supports per-directory configurations specified in CPPLINT.cfg
- files. CPPLINT.cfg file can contain a number of key=value pairs.
- Currently the following options are supported:
-
- set noparent
- filter=+filter1,-filter2,...
- exclude_files=regex
- linelength=80
-
- "set noparent" option prevents cpplint from traversing directory tree
- upwards looking for more .cfg files in parent directories. This option
- is usually placed in the top-level project directory.
-
- The "filter" option is similar in function to --filter flag. It specifies
- message filters in addition to the |_DEFAULT_FILTERS| and those specified
- through --filter command-line flag.
-
- "exclude_files" allows to specify a regular expression to be matched against
- a file name. If the expression matches, the file is skipped and not run
- through liner.
-
- "linelength" allows to specify the allowed line length for the project.
-
- CPPLINT.cfg has an effect on files in the same directory and all
- sub-directories, unless overridden by a nested configuration file.
-
- Example file:
- filter=-build/include_order,+build/include_alpha
- exclude_files=.*\.cc
-
- The above example disables build/include_order warning and enables
- build/include_alpha as well as excludes all .cc from being
- processed by linter, in the current directory (where the .cfg
- file is located) and all sub-directories.
-"""
-
-# We categorize each error message we print. Here are the categories.
-# We want an explicit list so we can list them all in cpplint --filter=.
-# If you add a new error message with a new category, add it to the list
-# here! cpplint_unittest.py should tell you if you forget to do this.
-_ERROR_CATEGORIES = [
- 'build/class',
- 'build/c++11',
- 'build/deprecated',
- 'build/endif_comment',
- 'build/explicit_make_pair',
- 'build/forward_decl',
- 'build/header_guard',
- 'build/include',
- 'build/include_alpha',
- 'build/include_order',
- 'build/include_what_you_use',
- 'build/namespaces',
- 'build/printf_format',
- 'build/storage_class',
- 'legal/copyright',
- 'readability/alt_tokens',
- 'readability/braces',
- 'readability/casting',
- 'readability/check',
- 'readability/constructors',
- 'readability/fn_size',
- 'readability/function',
- 'readability/inheritance',
- 'readability/multiline_comment',
- 'readability/multiline_string',
- 'readability/namespace',
- 'readability/nolint',
- 'readability/nul',
- 'readability/strings',
- 'readability/todo',
- 'readability/utf8',
- 'runtime/arrays',
- 'runtime/casting',
- 'runtime/explicit',
- 'runtime/int',
- 'runtime/init',
- 'runtime/invalid_increment',
- 'runtime/member_string_references',
- 'runtime/memset',
- 'runtime/indentation_namespace',
- 'runtime/operator',
- 'runtime/printf',
- 'runtime/printf_format',
- 'runtime/references',
- 'runtime/string',
- 'runtime/threadsafe_fn',
- 'runtime/vlog',
- 'whitespace/blank_line',
- 'whitespace/braces',
- 'whitespace/comma',
- 'whitespace/comments',
- 'whitespace/empty_conditional_body',
- 'whitespace/empty_loop_body',
- 'whitespace/end_of_line',
- 'whitespace/ending_newline',
- 'whitespace/forcolon',
- 'whitespace/indent',
- 'whitespace/line_length',
- 'whitespace/newline',
- 'whitespace/operators',
- 'whitespace/parens',
- 'whitespace/semicolon',
- 'whitespace/tab',
- 'whitespace/todo',
- ]
-
-# These error categories are no longer enforced by cpplint, but for backwards-
-# compatibility they may still appear in NOLINT comments.
-_LEGACY_ERROR_CATEGORIES = [
- 'readability/streams',
- ]
-
-# The default state of the category filter. This is overridden by the --filter=
-# flag. By default all errors are on, so only add here categories that should be
-# off by default (i.e., categories that must be enabled by the --filter= flags).
-# All entries here should start with a '-' or '+', as in the --filter= flag.
-_DEFAULT_FILTERS = ['-build/include_alpha']
-
-# We used to check for high-bit characters, but after much discussion we
-# decided those were OK, as long as they were in UTF-8 and didn't represent
-# hard-coded international strings, which belong in a separate i18n file.
-
-# C++ headers
-_CPP_HEADERS = frozenset([
- # Legacy
- 'algobase.h',
- 'algo.h',
- 'alloc.h',
- 'builtinbuf.h',
- 'bvector.h',
- 'complex.h',
- 'defalloc.h',
- 'deque.h',
- 'editbuf.h',
- 'fstream.h',
- 'function.h',
- 'hash_map',
- 'hash_map.h',
- 'hash_set',
- 'hash_set.h',
- 'hashtable.h',
- 'heap.h',
- 'indstream.h',
- 'iomanip.h',
- 'iostream.h',
- 'istream.h',
- 'iterator.h',
- 'list.h',
- 'map.h',
- 'multimap.h',
- 'multiset.h',
- 'ostream.h',
- 'pair.h',
- 'parsestream.h',
- 'pfstream.h',
- 'procbuf.h',
- 'pthread_alloc',
- 'pthread_alloc.h',
- 'rope',
- 'rope.h',
- 'ropeimpl.h',
- 'set.h',
- 'slist',
- 'slist.h',
- 'stack.h',
- 'stdiostream.h',
- 'stl_alloc.h',
- 'stl_relops.h',
- 'streambuf.h',
- 'stream.h',
- 'strfile.h',
- 'strstream.h',
- 'tempbuf.h',
- 'tree.h',
- 'type_traits.h',
- 'vector.h',
- # 17.6.1.2 C++ library headers
- 'algorithm',
- 'array',
- 'atomic',
- 'bitset',
- 'chrono',
- 'codecvt',
- 'complex',
- 'condition_variable',
- 'deque',
- 'exception',
- 'forward_list',
- 'fstream',
- 'functional',
- 'future',
- 'initializer_list',
- 'iomanip',
- 'ios',
- 'iosfwd',
- 'iostream',
- 'istream',
- 'iterator',
- 'limits',
- 'list',
- 'locale',
- 'map',
- 'memory',
- 'mutex',
- 'new',
- 'numeric',
- 'ostream',
- 'queue',
- 'random',
- 'ratio',
- 'regex',
- 'set',
- 'sstream',
- 'stack',
- 'stdexcept',
- 'streambuf',
- 'string',
- 'strstream',
- 'system_error',
- 'thread',
- 'tuple',
- 'typeindex',
- 'typeinfo',
- 'type_traits',
- 'unordered_map',
- 'unordered_set',
- 'utility',
- 'valarray',
- 'vector',
- # 17.6.1.2 C++ headers for C library facilities
- 'cassert',
- 'ccomplex',
- 'cctype',
- 'cerrno',
- 'cfenv',
- 'cfloat',
- 'cinttypes',
- 'ciso646',
- 'climits',
- 'clocale',
- 'cmath',
- 'csetjmp',
- 'csignal',
- 'cstdalign',
- 'cstdarg',
- 'cstdbool',
- 'cstddef',
- 'cstdint',
- 'cstdio',
- 'cstdlib',
- 'cstring',
- 'ctgmath',
- 'ctime',
- 'cuchar',
- 'cwchar',
- 'cwctype',
- ])
-
-
-# These headers are excluded from [build/include] and [build/include_order]
-# checks:
-# - Anything not following google file name conventions (containing an
-# uppercase character, such as Python.h or nsStringAPI.h, for example).
-# - Lua headers.
-_THIRD_PARTY_HEADERS_PATTERN = re.compile(
- r'^(?:[^/]*[A-Z][^/]*\.h|lua\.h|lauxlib\.h|lualib\.h)$')
-
-
-# Assertion macros. These are defined in base/logging.h and
-# testing/base/gunit.h. Note that the _M versions need to come first
-# for substring matching to work.
-_CHECK_MACROS = [
- 'DCHECK', 'CHECK',
- 'EXPECT_TRUE_M', 'EXPECT_TRUE',
- 'ASSERT_TRUE_M', 'ASSERT_TRUE',
- 'EXPECT_FALSE_M', 'EXPECT_FALSE',
- 'ASSERT_FALSE_M', 'ASSERT_FALSE',
- ]
-
-# Replacement macros for CHECK/DCHECK/EXPECT_TRUE/EXPECT_FALSE
-_CHECK_REPLACEMENT = dict([(m, {}) for m in _CHECK_MACROS])
-
-for op, replacement in [('==', 'EQ'), ('!=', 'NE'),
- ('>=', 'GE'), ('>', 'GT'),
- ('<=', 'LE'), ('<', 'LT')]:
- _CHECK_REPLACEMENT['DCHECK'][op] = 'DCHECK_%s' % replacement
- _CHECK_REPLACEMENT['CHECK'][op] = 'CHECK_%s' % replacement
- _CHECK_REPLACEMENT['EXPECT_TRUE'][op] = 'EXPECT_%s' % replacement
- _CHECK_REPLACEMENT['ASSERT_TRUE'][op] = 'ASSERT_%s' % replacement
- _CHECK_REPLACEMENT['EXPECT_TRUE_M'][op] = 'EXPECT_%s_M' % replacement
- _CHECK_REPLACEMENT['ASSERT_TRUE_M'][op] = 'ASSERT_%s_M' % replacement
-
-for op, inv_replacement in [('==', 'NE'), ('!=', 'EQ'),
- ('>=', 'LT'), ('>', 'LE'),
- ('<=', 'GT'), ('<', 'GE')]:
- _CHECK_REPLACEMENT['EXPECT_FALSE'][op] = 'EXPECT_%s' % inv_replacement
- _CHECK_REPLACEMENT['ASSERT_FALSE'][op] = 'ASSERT_%s' % inv_replacement
- _CHECK_REPLACEMENT['EXPECT_FALSE_M'][op] = 'EXPECT_%s_M' % inv_replacement
- _CHECK_REPLACEMENT['ASSERT_FALSE_M'][op] = 'ASSERT_%s_M' % inv_replacement
-
-# Alternative tokens and their replacements. For full list, see section 2.5
-# Alternative tokens [lex.digraph] in the C++ standard.
-#
-# Digraphs (such as '%:') are not included here since it's a mess to
-# match those on a word boundary.
-_ALT_TOKEN_REPLACEMENT = {
- 'and': '&&',
- 'bitor': '|',
- 'or': '||',
- 'xor': '^',
- 'compl': '~',
- 'bitand': '&',
- 'and_eq': '&=',
- 'or_eq': '|=',
- 'xor_eq': '^=',
- 'not': '!',
- 'not_eq': '!='
- }
-
-# Compile regular expression that matches all the above keywords. The "[ =()]"
-# bit is meant to avoid matching these keywords outside of boolean expressions.
-#
-# False positives include C-style multi-line comments and multi-line strings
-# but those have always been troublesome for cpplint.
-_ALT_TOKEN_REPLACEMENT_PATTERN = re.compile(
- r'[ =()](' + ('|'.join(_ALT_TOKEN_REPLACEMENT.keys())) + r')(?=[ (]|$)')
-
-
-# These constants define types of headers for use with
-# _IncludeState.CheckNextIncludeOrder().
-_C_SYS_HEADER = 1
-_CPP_SYS_HEADER = 2
-_LIKELY_MY_HEADER = 3
-_POSSIBLE_MY_HEADER = 4
-_OTHER_HEADER = 5
-
-# These constants define the current inline assembly state
-_NO_ASM = 0 # Outside of inline assembly block
-_INSIDE_ASM = 1 # Inside inline assembly block
-_END_ASM = 2 # Last line of inline assembly block
-_BLOCK_ASM = 3 # The whole block is an inline assembly block
-
-# Match start of assembly blocks
-_MATCH_ASM = re.compile(r'^\s*(?:asm|_asm|__asm|__asm__)'
- r'(?:\s+(volatile|__volatile__))?'
- r'\s*[{(]')
-
-
-_regexp_compile_cache = {}
-
-# {str, set(int)}: a map from error categories to sets of linenumbers
-# on which those errors are expected and should be suppressed.
-_error_suppressions = {}
-
-# The root directory used for deriving header guard CPP variable.
-# This is set by --root flag.
-_root = None
-
-# The allowed line length of files.
-# This is set by --linelength flag.
-_line_length = 80
-
-# The allowed extensions for file names
-# This is set by --extensions flag.
-_valid_extensions = set(['cc', 'h', 'cpp', 'cu', 'cuh'])
-
-def ParseNolintSuppressions(filename, raw_line, linenum, error):
- """Updates the global list of error-suppressions.
-
- Parses any NOLINT comments on the current line, updating the global
- error_suppressions store. Reports an error if the NOLINT comment
- was malformed.
-
- Args:
- filename: str, the name of the input file.
- raw_line: str, the line of input text, with comments.
- linenum: int, the number of the current line.
- error: function, an error handler.
- """
- matched = Search(r'\bNOLINT(NEXTLINE)?\b(\([^)]+\))?', raw_line)
- if matched:
- if matched.group(1):
- suppressed_line = linenum + 1
- else:
- suppressed_line = linenum
- category = matched.group(2)
- if category in (None, '(*)'): # => "suppress all"
- _error_suppressions.setdefault(None, set()).add(suppressed_line)
- else:
- if category.startswith('(') and category.endswith(')'):
- category = category[1:-1]
- if category in _ERROR_CATEGORIES:
- _error_suppressions.setdefault(category, set()).add(suppressed_line)
- elif category not in _LEGACY_ERROR_CATEGORIES:
- error(filename, linenum, 'readability/nolint', 5,
- 'Unknown NOLINT error category: %s' % category)
-
-
-def ResetNolintSuppressions():
- """Resets the set of NOLINT suppressions to empty."""
- _error_suppressions.clear()
-
-
-def IsErrorSuppressedByNolint(category, linenum):
- """Returns true if the specified error category is suppressed on this line.
-
- Consults the global error_suppressions map populated by
- ParseNolintSuppressions/ResetNolintSuppressions.
-
- Args:
- category: str, the category of the error.
- linenum: int, the current line number.
- Returns:
- bool, True iff the error should be suppressed due to a NOLINT comment.
- """
- return (linenum in _error_suppressions.get(category, set()) or
- linenum in _error_suppressions.get(None, set()))
-
-
-def Match(pattern, s):
- """Matches the string with the pattern, caching the compiled regexp."""
- # The regexp compilation caching is inlined in both Match and Search for
- # performance reasons; factoring it out into a separate function turns out
- # to be noticeably expensive.
- if pattern not in _regexp_compile_cache:
- _regexp_compile_cache[pattern] = sre_compile.compile(pattern)
- return _regexp_compile_cache[pattern].match(s)
-
-
-def ReplaceAll(pattern, rep, s):
- """Replaces instances of pattern in a string with a replacement.
-
- The compiled regex is kept in a cache shared by Match and Search.
-
- Args:
- pattern: regex pattern
- rep: replacement text
- s: search string
-
- Returns:
- string with replacements made (or original string if no replacements)
- """
- if pattern not in _regexp_compile_cache:
- _regexp_compile_cache[pattern] = sre_compile.compile(pattern)
- return _regexp_compile_cache[pattern].sub(rep, s)
-
-
-def Search(pattern, s):
- """Searches the string for the pattern, caching the compiled regexp."""
- if pattern not in _regexp_compile_cache:
- _regexp_compile_cache[pattern] = sre_compile.compile(pattern)
- return _regexp_compile_cache[pattern].search(s)
-
-
-class _IncludeState(object):
- """Tracks line numbers for includes, and the order in which includes appear.
-
- include_list contains list of lists of (header, line number) pairs.
- It's a lists of lists rather than just one flat list to make it
- easier to update across preprocessor boundaries.
-
- Call CheckNextIncludeOrder() once for each header in the file, passing
- in the type constants defined above. Calls in an illegal order will
- raise an _IncludeError with an appropriate error message.
-
- """
- # self._section will move monotonically through this set. If it ever
- # needs to move backwards, CheckNextIncludeOrder will raise an error.
- _INITIAL_SECTION = 0
- _MY_H_SECTION = 1
- _C_SECTION = 2
- _CPP_SECTION = 3
- _OTHER_H_SECTION = 4
-
- _TYPE_NAMES = {
- _C_SYS_HEADER: 'C system header',
- _CPP_SYS_HEADER: 'C++ system header',
- _LIKELY_MY_HEADER: 'header this file implements',
- _POSSIBLE_MY_HEADER: 'header this file may implement',
- _OTHER_HEADER: 'other header',
- }
- _SECTION_NAMES = {
- _INITIAL_SECTION: "... nothing. (This can't be an error.)",
- _MY_H_SECTION: 'a header this file implements',
- _C_SECTION: 'C system header',
- _CPP_SECTION: 'C++ system header',
- _OTHER_H_SECTION: 'other header',
- }
-
- def __init__(self):
- self.include_list = [[]]
- self.ResetSection('')
-
- def FindHeader(self, header):
- """Check if a header has already been included.
-
- Args:
- header: header to check.
- Returns:
- Line number of previous occurrence, or -1 if the header has not
- been seen before.
- """
- for section_list in self.include_list:
- for f in section_list:
- if f[0] == header:
- return f[1]
- return -1
-
- def ResetSection(self, directive):
- """Reset section checking for preprocessor directive.
-
- Args:
- directive: preprocessor directive (e.g. "if", "else").
- """
- # The name of the current section.
- self._section = self._INITIAL_SECTION
- # The path of last found header.
- self._last_header = ''
-
- # Update list of includes. Note that we never pop from the
- # include list.
- if directive in ('if', 'ifdef', 'ifndef'):
- self.include_list.append([])
- elif directive in ('else', 'elif'):
- self.include_list[-1] = []
-
- def SetLastHeader(self, header_path):
- self._last_header = header_path
-
- def CanonicalizeAlphabeticalOrder(self, header_path):
- """Returns a path canonicalized for alphabetical comparison.
-
- - replaces "-" with "_" so they both cmp the same.
- - removes '-inl' since we don't require them to be after the main header.
- - lowercase everything, just in case.
-
- Args:
- header_path: Path to be canonicalized.
-
- Returns:
- Canonicalized path.
- """
- return header_path.replace('-inl.h', '.h').replace('-', '_').lower()
-
- def IsInAlphabeticalOrder(self, clean_lines, linenum, header_path):
- """Check if a header is in alphabetical order with the previous header.
-
- Args:
- clean_lines: A CleansedLines instance containing the file.
- linenum: The number of the line to check.
- header_path: Canonicalized header to be checked.
-
- Returns:
- Returns true if the header is in alphabetical order.
- """
- # If previous section is different from current section, _last_header will
- # be reset to empty string, so it's always less than current header.
- #
- # If previous line was a blank line, assume that the headers are
- # intentionally sorted the way they are.
- if (self._last_header > header_path and
- Match(r'^\s*#\s*include\b', clean_lines.elided[linenum - 1])):
- return False
- return True
-
- def CheckNextIncludeOrder(self, header_type):
- """Returns a non-empty error message if the next header is out of order.
-
- This function also updates the internal state to be ready to check
- the next include.
-
- Args:
- header_type: One of the _XXX_HEADER constants defined above.
-
- Returns:
- The empty string if the header is in the right order, or an
- error message describing what's wrong.
-
- """
- error_message = ('Found %s after %s' %
- (self._TYPE_NAMES[header_type],
- self._SECTION_NAMES[self._section]))
-
- last_section = self._section
-
- if header_type == _C_SYS_HEADER:
- if self._section <= self._C_SECTION:
- self._section = self._C_SECTION
- else:
- self._last_header = ''
- return error_message
- elif header_type == _CPP_SYS_HEADER:
- if self._section <= self._CPP_SECTION:
- self._section = self._CPP_SECTION
- else:
- self._last_header = ''
- return error_message
- elif header_type == _LIKELY_MY_HEADER:
- if self._section <= self._MY_H_SECTION:
- self._section = self._MY_H_SECTION
- else:
- self._section = self._OTHER_H_SECTION
- elif header_type == _POSSIBLE_MY_HEADER:
- if self._section <= self._MY_H_SECTION:
- self._section = self._MY_H_SECTION
- else:
- # This will always be the fallback because we're not sure
- # enough that the header is associated with this file.
- self._section = self._OTHER_H_SECTION
- else:
- assert header_type == _OTHER_HEADER
- self._section = self._OTHER_H_SECTION
-
- if last_section != self._section:
- self._last_header = ''
-
- return ''
-
-
-class _CppLintState(object):
- """Maintains module-wide state.."""
-
- def __init__(self):
- self.verbose_level = 1 # global setting.
- self.error_count = 0 # global count of reported errors
- # filters to apply when emitting error messages
- self.filters = _DEFAULT_FILTERS[:]
- # backup of filter list. Used to restore the state after each file.
- self._filters_backup = self.filters[:]
- self.counting = 'total' # In what way are we counting errors?
- self.errors_by_category = {} # string to int dict storing error counts
-
- # output format:
- # "emacs" - format that emacs can parse (default)
- # "vs7" - format that Microsoft Visual Studio 7 can parse
- self.output_format = 'emacs'
-
- def SetOutputFormat(self, output_format):
- """Sets the output format for errors."""
- self.output_format = output_format
-
- def SetVerboseLevel(self, level):
- """Sets the module's verbosity, and returns the previous setting."""
- last_verbose_level = self.verbose_level
- self.verbose_level = level
- return last_verbose_level
-
- def SetCountingStyle(self, counting_style):
- """Sets the module's counting options."""
- self.counting = counting_style
-
- def SetFilters(self, filters):
- """Sets the error-message filters.
-
- These filters are applied when deciding whether to emit a given
- error message.
-
- Args:
- filters: A string of comma-separated filters (eg "+whitespace/indent").
- Each filter should start with + or -; else we die.
-
- Raises:
- ValueError: The comma-separated filters did not all start with '+' or '-'.
- E.g. "-,+whitespace,-whitespace/indent,whitespace/badfilter"
- """
- # Default filters always have less priority than the flag ones.
- self.filters = _DEFAULT_FILTERS[:]
- self.AddFilters(filters)
-
- def AddFilters(self, filters):
- """ Adds more filters to the existing list of error-message filters. """
- for filt in filters.split(','):
- clean_filt = filt.strip()
- if clean_filt:
- self.filters.append(clean_filt)
- for filt in self.filters:
- if not (filt.startswith('+') or filt.startswith('-')):
- raise ValueError('Every filter in --filters must start with + or -'
- ' (%s does not)' % filt)
-
- def BackupFilters(self):
- """ Saves the current filter list to backup storage."""
- self._filters_backup = self.filters[:]
-
- def RestoreFilters(self):
- """ Restores filters previously backed up."""
- self.filters = self._filters_backup[:]
-
- def ResetErrorCounts(self):
- """Sets the module's error statistic back to zero."""
- self.error_count = 0
- self.errors_by_category = {}
-
- def IncrementErrorCount(self, category):
- """Bumps the module's error statistic."""
- self.error_count += 1
- if self.counting in ('toplevel', 'detailed'):
- if self.counting != 'detailed':
- category = category.split('/')[0]
- if category not in self.errors_by_category:
- self.errors_by_category[category] = 0
- self.errors_by_category[category] += 1
-
- def PrintErrorCounts(self):
- """Print a summary of errors by category, and the total."""
- for category, count in self.errors_by_category.iteritems():
- sys.stderr.write('Category \'%s\' errors found: %d\n' %
- (category, count))
- sys.stderr.write('Total errors found: %d\n' % self.error_count)
-
-_cpplint_state = _CppLintState()
-
-
-def _OutputFormat():
- """Gets the module's output format."""
- return _cpplint_state.output_format
-
-
-def _SetOutputFormat(output_format):
- """Sets the module's output format."""
- _cpplint_state.SetOutputFormat(output_format)
-
-
-def _VerboseLevel():
- """Returns the module's verbosity setting."""
- return _cpplint_state.verbose_level
-
-
-def _SetVerboseLevel(level):
- """Sets the module's verbosity, and returns the previous setting."""
- return _cpplint_state.SetVerboseLevel(level)
-
-
-def _SetCountingStyle(level):
- """Sets the module's counting options."""
- _cpplint_state.SetCountingStyle(level)
-
-
-def _Filters():
- """Returns the module's list of output filters, as a list."""
- return _cpplint_state.filters
-
-
-def _SetFilters(filters):
- """Sets the module's error-message filters.
-
- These filters are applied when deciding whether to emit a given
- error message.
-
- Args:
- filters: A string of comma-separated filters (eg "whitespace/indent").
- Each filter should start with + or -; else we die.
- """
- _cpplint_state.SetFilters(filters)
-
-def _AddFilters(filters):
- """Adds more filter overrides.
-
- Unlike _SetFilters, this function does not reset the current list of filters
- available.
-
- Args:
- filters: A string of comma-separated filters (eg "whitespace/indent").
- Each filter should start with + or -; else we die.
- """
- _cpplint_state.AddFilters(filters)
-
-def _BackupFilters():
- """ Saves the current filter list to backup storage."""
- _cpplint_state.BackupFilters()
-
-def _RestoreFilters():
- """ Restores filters previously backed up."""
- _cpplint_state.RestoreFilters()
-
-class _FunctionState(object):
- """Tracks current function name and the number of lines in its body."""
-
- _NORMAL_TRIGGER = 250 # for --v=0, 500 for --v=1, etc.
- _TEST_TRIGGER = 400 # about 50% more than _NORMAL_TRIGGER.
-
- def __init__(self):
- self.in_a_function = False
- self.lines_in_function = 0
- self.current_function = ''
-
- def Begin(self, function_name):
- """Start analyzing function body.
-
- Args:
- function_name: The name of the function being tracked.
- """
- self.in_a_function = True
- self.lines_in_function = 0
- self.current_function = function_name
-
- def Count(self):
- """Count line in current function body."""
- if self.in_a_function:
- self.lines_in_function += 1
-
- def Check(self, error, filename, linenum):
- """Report if too many lines in function body.
-
- Args:
- error: The function to call with any errors found.
- filename: The name of the current file.
- linenum: The number of the line to check.
- """
- if Match(r'T(EST|est)', self.current_function):
- base_trigger = self._TEST_TRIGGER
- else:
- base_trigger = self._NORMAL_TRIGGER
- trigger = base_trigger * 2**_VerboseLevel()
-
- if self.lines_in_function > trigger:
- error_level = int(math.log(self.lines_in_function / base_trigger, 2))
- # 50 => 0, 100 => 1, 200 => 2, 400 => 3, 800 => 4, 1600 => 5, ...
- if error_level > 5:
- error_level = 5
- error(filename, linenum, 'readability/fn_size', error_level,
- 'Small and focused functions are preferred:'
- ' %s has %d non-comment lines'
- ' (error triggered by exceeding %d lines).' % (
- self.current_function, self.lines_in_function, trigger))
-
- def End(self):
- """Stop analyzing function body."""
- self.in_a_function = False
-
-
-class _IncludeError(Exception):
- """Indicates a problem with the include order in a file."""
- pass
-
-
-class FileInfo(object):
- """Provides utility functions for filenames.
-
- FileInfo provides easy access to the components of a file's path
- relative to the project root.
- """
-
- def __init__(self, filename):
- self._filename = filename
-
- def FullName(self):
- """Make Windows paths like Unix."""
- return os.path.abspath(self._filename).replace('\\', '/')
-
- def RepositoryName(self):
- """FullName after removing the local path to the repository.
-
- If we have a real absolute path name here we can try to do something smart:
- detecting the root of the checkout and truncating /path/to/checkout from
- the name so that we get header guards that don't include things like
- "C:\Documents and Settings\..." or "/home/username/..." in them and thus
- people on different computers who have checked the source out to different
- locations won't see bogus errors.
- """
- fullname = self.FullName()
-
- if os.path.exists(fullname):
- project_dir = os.path.dirname(fullname)
-
- if os.path.exists(os.path.join(project_dir, ".svn")):
- # If there's a .svn file in the current directory, we recursively look
- # up the directory tree for the top of the SVN checkout
- root_dir = project_dir
- one_up_dir = os.path.dirname(root_dir)
- while os.path.exists(os.path.join(one_up_dir, ".svn")):
- root_dir = os.path.dirname(root_dir)
- one_up_dir = os.path.dirname(one_up_dir)
-
- prefix = os.path.commonprefix([root_dir, project_dir])
- return fullname[len(prefix) + 1:]
-
- # Not SVN <= 1.6? Try to find a git, hg, or svn top level directory by
- # searching up from the current path.
- root_dir = os.path.dirname(fullname)
- while (root_dir != os.path.dirname(root_dir) and
- not os.path.exists(os.path.join(root_dir, ".git")) and
- not os.path.exists(os.path.join(root_dir, ".hg")) and
- not os.path.exists(os.path.join(root_dir, ".svn"))):
- root_dir = os.path.dirname(root_dir)
-
- if (os.path.exists(os.path.join(root_dir, ".git")) or
- os.path.exists(os.path.join(root_dir, ".hg")) or
- os.path.exists(os.path.join(root_dir, ".svn"))):
- prefix = os.path.commonprefix([root_dir, project_dir])
- return fullname[len(prefix) + 1:]
-
- # Don't know what to do; header guard warnings may be wrong...
- return fullname
-
- def Split(self):
- """Splits the file into the directory, basename, and extension.
-
- For 'chrome/browser/browser.cc', Split() would
- return ('chrome/browser', 'browser', '.cc')
-
- Returns:
- A tuple of (directory, basename, extension).
- """
-
- googlename = self.RepositoryName()
- project, rest = os.path.split(googlename)
- return (project,) + os.path.splitext(rest)
-
- def BaseName(self):
- """File base name - text after the final slash, before the final period."""
- return self.Split()[1]
-
- def Extension(self):
- """File extension - text following the final period."""
- return self.Split()[2]
-
- def NoExtension(self):
- """File has no source file extension."""
- return '/'.join(self.Split()[0:2])
-
- def IsSource(self):
- """File has a source file extension."""
- return self.Extension()[1:] in ('c', 'cc', 'cpp', 'cxx')
-
-
-def _ShouldPrintError(category, confidence, linenum):
- """If confidence >= verbose, category passes filter and is not suppressed."""
-
- # There are three ways we might decide not to print an error message:
- # a "NOLINT(category)" comment appears in the source,
- # the verbosity level isn't high enough, or the filters filter it out.
- if IsErrorSuppressedByNolint(category, linenum):
- return False
-
- if confidence < _cpplint_state.verbose_level:
- return False
-
- is_filtered = False
- for one_filter in _Filters():
- if one_filter.startswith('-'):
- if category.startswith(one_filter[1:]):
- is_filtered = True
- elif one_filter.startswith('+'):
- if category.startswith(one_filter[1:]):
- is_filtered = False
- else:
- assert False # should have been checked for in SetFilter.
- if is_filtered:
- return False
-
- return True
-
-
-def Error(filename, linenum, category, confidence, message):
- """Logs the fact we've found a lint error.
-
- We log where the error was found, and also our confidence in the error,
- that is, how certain we are this is a legitimate style regression, and
- not a misidentification or a use that's sometimes justified.
-
- False positives can be suppressed by the use of
- "cpplint(category)" comments on the offending line. These are
- parsed into _error_suppressions.
-
- Args:
- filename: The name of the file containing the error.
- linenum: The number of the line containing the error.
- category: A string used to describe the "category" this bug
- falls under: "whitespace", say, or "runtime". Categories
- may have a hierarchy separated by slashes: "whitespace/indent".
- confidence: A number from 1-5 representing a confidence score for
- the error, with 5 meaning that we are certain of the problem,
- and 1 meaning that it could be a legitimate construct.
- message: The error message.
- """
- if _ShouldPrintError(category, confidence, linenum):
- _cpplint_state.IncrementErrorCount(category)
- if _cpplint_state.output_format == 'vs7':
- sys.stderr.write('%s(%s): %s [%s] [%d]\n' % (
- filename, linenum, message, category, confidence))
- elif _cpplint_state.output_format == 'eclipse':
- sys.stderr.write('%s:%s: warning: %s [%s] [%d]\n' % (
- filename, linenum, message, category, confidence))
- else:
- sys.stderr.write('%s:%s: %s [%s] [%d]\n' % (
- filename, linenum, message, category, confidence))
-
-
-# Matches standard C++ escape sequences per 2.13.2.3 of the C++ standard.
-_RE_PATTERN_CLEANSE_LINE_ESCAPES = re.compile(
- r'\\([abfnrtv?"\\\']|\d+|x[0-9a-fA-F]+)')
-# Match a single C style comment on the same line.
-_RE_PATTERN_C_COMMENTS = r'/\*(?:[^*]|\*(?!/))*\*/'
-# Matches multi-line C style comments.
-# This RE is a little bit more complicated than one might expect, because we
-# have to take care of space removals tools so we can handle comments inside
-# statements better.
-# The current rule is: We only clear spaces from both sides when we're at the
-# end of the line. Otherwise, we try to remove spaces from the right side,
-# if this doesn't work we try on left side but only if there's a non-character
-# on the right.
-_RE_PATTERN_CLEANSE_LINE_C_COMMENTS = re.compile(
- r'(\s*' + _RE_PATTERN_C_COMMENTS + r'\s*$|' +
- _RE_PATTERN_C_COMMENTS + r'\s+|' +
- r'\s+' + _RE_PATTERN_C_COMMENTS + r'(?=\W)|' +
- _RE_PATTERN_C_COMMENTS + r')')
-
-
-def IsCppString(line):
- """Does line terminate so, that the next symbol is in string constant.
-
- This function does not consider single-line nor multi-line comments.
-
- Args:
- line: is a partial line of code starting from the 0..n.
-
- Returns:
- True, if next character appended to 'line' is inside a
- string constant.
- """
-
- line = line.replace(r'\\', 'XX') # after this, \\" does not match to \"
- return ((line.count('"') - line.count(r'\"') - line.count("'\"'")) & 1) == 1
-
-
-def CleanseRawStrings(raw_lines):
- """Removes C++11 raw strings from lines.
-
- Before:
- static const char kData[] = R"(
- multi-line string
- )";
-
- After:
- static const char kData[] = ""
- (replaced by blank line)
- "";
-
- Args:
- raw_lines: list of raw lines.
-
- Returns:
- list of lines with C++11 raw strings replaced by empty strings.
- """
-
- delimiter = None
- lines_without_raw_strings = []
- for line in raw_lines:
- if delimiter:
- # Inside a raw string, look for the end
- end = line.find(delimiter)
- if end >= 0:
- # Found the end of the string, match leading space for this
- # line and resume copying the original lines, and also insert
- # a "" on the last line.
- leading_space = Match(r'^(\s*)\S', line)
- line = leading_space.group(1) + '""' + line[end + len(delimiter):]
- delimiter = None
- else:
- # Haven't found the end yet, append a blank line.
- line = '""'
-
- # Look for beginning of a raw string, and replace them with
- # empty strings. This is done in a loop to handle multiple raw
- # strings on the same line.
- while delimiter is None:
- # Look for beginning of a raw string.
- # See 2.14.15 [lex.string] for syntax.
- matched = Match(r'^(.*)\b(?:R|u8R|uR|UR|LR)"([^\s\\()]*)\((.*)$', line)
- if matched:
- delimiter = ')' + matched.group(2) + '"'
-
- end = matched.group(3).find(delimiter)
- if end >= 0:
- # Raw string ended on same line
- line = (matched.group(1) + '""' +
- matched.group(3)[end + len(delimiter):])
- delimiter = None
- else:
- # Start of a multi-line raw string
- line = matched.group(1) + '""'
- else:
- break
-
- lines_without_raw_strings.append(line)
-
- # TODO(unknown): if delimiter is not None here, we might want to
- # emit a warning for unterminated string.
- return lines_without_raw_strings
-
-
-def FindNextMultiLineCommentStart(lines, lineix):
- """Find the beginning marker for a multiline comment."""
- while lineix < len(lines):
- if lines[lineix].strip().startswith('/*'):
- # Only return this marker if the comment goes beyond this line
- if lines[lineix].strip().find('*/', 2) < 0:
- return lineix
- lineix += 1
- return len(lines)
-
-
-def FindNextMultiLineCommentEnd(lines, lineix):
- """We are inside a comment, find the end marker."""
- while lineix < len(lines):
- if lines[lineix].strip().endswith('*/'):
- return lineix
- lineix += 1
- return len(lines)
-
-
-def RemoveMultiLineCommentsFromRange(lines, begin, end):
- """Clears a range of lines for multi-line comments."""
- # Having // dummy comments makes the lines non-empty, so we will not get
- # unnecessary blank line warnings later in the code.
- for i in range(begin, end):
- lines[i] = '/**/'
-
-
-def RemoveMultiLineComments(filename, lines, error):
- """Removes multiline (c-style) comments from lines."""
- lineix = 0
- while lineix < len(lines):
- lineix_begin = FindNextMultiLineCommentStart(lines, lineix)
- if lineix_begin >= len(lines):
- return
- lineix_end = FindNextMultiLineCommentEnd(lines, lineix_begin)
- if lineix_end >= len(lines):
- error(filename, lineix_begin + 1, 'readability/multiline_comment', 5,
- 'Could not find end of multi-line comment')
- return
- RemoveMultiLineCommentsFromRange(lines, lineix_begin, lineix_end + 1)
- lineix = lineix_end + 1
-
-
-def CleanseComments(line):
- """Removes //-comments and single-line C-style /* */ comments.
-
- Args:
- line: A line of C++ source.
-
- Returns:
- The line with single-line comments removed.
- """
- commentpos = line.find('//')
- if commentpos != -1 and not IsCppString(line[:commentpos]):
- line = line[:commentpos].rstrip()
- # get rid of /* ... */
- return _RE_PATTERN_CLEANSE_LINE_C_COMMENTS.sub('', line)
-
-
-class CleansedLines(object):
- """Holds 4 copies of all lines with different preprocessing applied to them.
-
- 1) elided member contains lines without strings and comments.
- 2) lines member contains lines without comments.
- 3) raw_lines member contains all the lines without processing.
- 4) lines_without_raw_strings member is same as raw_lines, but with C++11 raw
- strings removed.
- All these members are of <type 'list'>, and of the same length.
- """
-
- def __init__(self, lines):
- self.elided = []
- self.lines = []
- self.raw_lines = lines
- self.num_lines = len(lines)
- self.lines_without_raw_strings = CleanseRawStrings(lines)
- for linenum in range(len(self.lines_without_raw_strings)):
- self.lines.append(CleanseComments(
- self.lines_without_raw_strings[linenum]))
- elided = self._CollapseStrings(self.lines_without_raw_strings[linenum])
- self.elided.append(CleanseComments(elided))
-
- def NumLines(self):
- """Returns the number of lines represented."""
- return self.num_lines
-
- @staticmethod
- def _CollapseStrings(elided):
- """Collapses strings and chars on a line to simple "" or '' blocks.
-
- We nix strings first so we're not fooled by text like '"http://"'
-
- Args:
- elided: The line being processed.
-
- Returns:
- The line with collapsed strings.
- """
- if _RE_PATTERN_INCLUDE.match(elided):
- return elided
-
- # Remove escaped characters first to make quote/single quote collapsing
- # basic. Things that look like escaped characters shouldn't occur
- # outside of strings and chars.
- elided = _RE_PATTERN_CLEANSE_LINE_ESCAPES.sub('', elided)
-
- # Replace quoted strings and digit separators. Both single quotes
- # and double quotes are processed in the same loop, otherwise
- # nested quotes wouldn't work.
- collapsed = ''
- while True:
- # Find the first quote character
- match = Match(r'^([^\'"]*)([\'"])(.*)$', elided)
- if not match:
- collapsed += elided
- break
- head, quote, tail = match.groups()
-
- if quote == '"':
- # Collapse double quoted strings
- second_quote = tail.find('"')
- if second_quote >= 0:
- collapsed += head + '""'
- elided = tail[second_quote + 1:]
- else:
- # Unmatched double quote, don't bother processing the rest
- # of the line since this is probably a multiline string.
- collapsed += elided
- break
- else:
- # Found single quote, check nearby text to eliminate digit separators.
- #
- # There is no special handling for floating point here, because
- # the integer/fractional/exponent parts would all be parsed
- # correctly as long as there are digits on both sides of the
- # separator. So we are fine as long as we don't see something
- # like "0.'3" (gcc 4.9.0 will not allow this literal).
- if Search(r'\b(?:0[bBxX]?|[1-9])[0-9a-fA-F]*$', head):
- match_literal = Match(r'^((?:\'?[0-9a-zA-Z_])*)(.*)$', "'" + tail)
- collapsed += head + match_literal.group(1).replace("'", '')
- elided = match_literal.group(2)
- else:
- second_quote = tail.find('\'')
- if second_quote >= 0:
- collapsed += head + "''"
- elided = tail[second_quote + 1:]
- else:
- # Unmatched single quote
- collapsed += elided
- break
-
- return collapsed
-
-
-def FindEndOfExpressionInLine(line, startpos, stack):
- """Find the position just after the end of current parenthesized expression.
-
- Args:
- line: a CleansedLines line.
- startpos: start searching at this position.
- stack: nesting stack at startpos.
-
- Returns:
- On finding matching end: (index just after matching end, None)
- On finding an unclosed expression: (-1, None)
- Otherwise: (-1, new stack at end of this line)
- """
- for i in xrange(startpos, len(line)):
- char = line[i]
- if char in '([{':
- # Found start of parenthesized expression, push to expression stack
- stack.append(char)
- elif char == '<':
- # Found potential start of template argument list
- if i > 0 and line[i - 1] == '<':
- # Left shift operator
- if stack and stack[-1] == '<':
- stack.pop()
- if not stack:
- return (-1, None)
- elif i > 0 and Search(r'\boperator\s*$', line[0:i]):
- # operator<, don't add to stack
- continue
- else:
- # Tentative start of template argument list
- stack.append('<')
- elif char in ')]}':
- # Found end of parenthesized expression.
- #
- # If we are currently expecting a matching '>', the pending '<'
- # must have been an operator. Remove them from expression stack.
- while stack and stack[-1] == '<':
- stack.pop()
- if not stack:
- return (-1, None)
- if ((stack[-1] == '(' and char == ')') or
- (stack[-1] == '[' and char == ']') or
- (stack[-1] == '{' and char == '}')):
- stack.pop()
- if not stack:
- return (i + 1, None)
- else:
- # Mismatched parentheses
- return (-1, None)
- elif char == '>':
- # Found potential end of template argument list.
-
- # Ignore "->" and operator functions
- if (i > 0 and
- (line[i - 1] == '-' or Search(r'\boperator\s*$', line[0:i - 1]))):
- continue
-
- # Pop the stack if there is a matching '<'. Otherwise, ignore
- # this '>' since it must be an operator.
- if stack:
- if stack[-1] == '<':
- stack.pop()
- if not stack:
- return (i + 1, None)
- elif char == ';':
- # Found something that look like end of statements. If we are currently
- # expecting a '>', the matching '<' must have been an operator, since
- # template argument list should not contain statements.
- while stack and stack[-1] == '<':
- stack.pop()
- if not stack:
- return (-1, None)
-
- # Did not find end of expression or unbalanced parentheses on this line
- return (-1, stack)
-
-
-def CloseExpression(clean_lines, linenum, pos):
- """If input points to ( or { or [ or <, finds the position that closes it.
-
- If lines[linenum][pos] points to a '(' or '{' or '[' or '<', finds the
- linenum/pos that correspond to the closing of the expression.
-
- TODO(unknown): cpplint spends a fair bit of time matching parentheses.
- Ideally we would want to index all opening and closing parentheses once
- and have CloseExpression be just a simple lookup, but due to preprocessor
- tricks, this is not so easy.
-
- Args:
- clean_lines: A CleansedLines instance containing the file.
- linenum: The number of the line to check.
- pos: A position on the line.
-
- Returns:
- A tuple (line, linenum, pos) pointer *past* the closing brace, or
- (line, len(lines), -1) if we never find a close. Note we ignore
- strings and comments when matching; and the line we return is the
- 'cleansed' line at linenum.
- """
-
- line = clean_lines.elided[linenum]
- if (line[pos] not in '({[<') or Match(r'<[<=]', line[pos:]):
- return (line, clean_lines.NumLines(), -1)
-
- # Check first line
- (end_pos, stack) = FindEndOfExpressionInLine(line, pos, [])
- if end_pos > -1:
- return (line, linenum, end_pos)
-
- # Continue scanning forward
- while stack and linenum < clean_lines.NumLines() - 1:
- linenum += 1
- line = clean_lines.elided[linenum]
- (end_pos, stack) = FindEndOfExpressionInLine(line, 0, stack)
- if end_pos > -1:
- return (line, linenum, end_pos)
-
- # Did not find end of expression before end of file, give up
- return (line, clean_lines.NumLines(), -1)
-
-
-def FindStartOfExpressionInLine(line, endpos, stack):
- """Find position at the matching start of current expression.
-
- This is almost the reverse of FindEndOfExpressionInLine, but note
- that the input position and returned position differs by 1.
-
- Args:
- line: a CleansedLines line.
- endpos: start searching at this position.
- stack: nesting stack at endpos.
-
- Returns:
- On finding matching start: (index at matching start, None)
- On finding an unclosed expression: (-1, None)
- Otherwise: (-1, new stack at beginning of this line)
- """
- i = endpos
- while i >= 0:
- char = line[i]
- if char in ')]}':
- # Found end of expression, push to expression stack
- stack.append(char)
- elif char == '>':
- # Found potential end of template argument list.
- #
- # Ignore it if it's a "->" or ">=" or "operator>"
- if (i > 0 and
- (line[i - 1] == '-' or
- Match(r'\s>=\s', line[i - 1:]) or
- Search(r'\boperator\s*$', line[0:i]))):
- i -= 1
- else:
- stack.append('>')
- elif char == '<':
- # Found potential start of template argument list
- if i > 0 and line[i - 1] == '<':
- # Left shift operator
- i -= 1
- else:
- # If there is a matching '>', we can pop the expression stack.
- # Otherwise, ignore this '<' since it must be an operator.
- if stack and stack[-1] == '>':
- stack.pop()
- if not stack:
- return (i, None)
- elif char in '([{':
- # Found start of expression.
- #
- # If there are any unmatched '>' on the stack, they must be
- # operators. Remove those.
- while stack and stack[-1] == '>':
- stack.pop()
- if not stack:
- return (-1, None)
- if ((char == '(' and stack[-1] == ')') or
- (char == '[' and stack[-1] == ']') or
- (char == '{' and stack[-1] == '}')):
- stack.pop()
- if not stack:
- return (i, None)
- else:
- # Mismatched parentheses
- return (-1, None)
- elif char == ';':
- # Found something that look like end of statements. If we are currently
- # expecting a '<', the matching '>' must have been an operator, since
- # template argument list should not contain statements.
- while stack and stack[-1] == '>':
- stack.pop()
- if not stack:
- return (-1, None)
-
- i -= 1
-
- return (-1, stack)
-
-
-def ReverseCloseExpression(clean_lines, linenum, pos):
- """If input points to ) or } or ] or >, finds the position that opens it.
-
- If lines[linenum][pos] points to a ')' or '}' or ']' or '>', finds the
- linenum/pos that correspond to the opening of the expression.
-
- Args:
- clean_lines: A CleansedLines instance containing the file.
- linenum: The number of the line to check.
- pos: A position on the line.
-
- Returns:
- A tuple (line, linenum, pos) pointer *at* the opening brace, or
- (line, 0, -1) if we never find the matching opening brace. Note
- we ignore strings and comments when matching; and the line we
- return is the 'cleansed' line at linenum.
- """
- line = clean_lines.elided[linenum]
- if line[pos] not in ')}]>':
- return (line, 0, -1)
-
- # Check last line
- (start_pos, stack) = FindStartOfExpressionInLine(line, pos, [])
- if start_pos > -1:
- return (line, linenum, start_pos)
-
- # Continue scanning backward
- while stack and linenum > 0:
- linenum -= 1
- line = clean_lines.elided[linenum]
- (start_pos, stack) = FindStartOfExpressionInLine(line, len(line) - 1, stack)
- if start_pos > -1:
- return (line, linenum, start_pos)
-
- # Did not find start of expression before beginning of file, give up
- return (line, 0, -1)
-
-
-def CheckForCopyright(filename, lines, error):
- """Logs an error if no Copyright message appears at the top of the file."""
-
- # We'll say it should occur by line 10. Don't forget there's a
- # dummy line at the front.
- for line in xrange(1, min(len(lines), 11)):
- if re.search(r'Copyright', lines[line], re.I): break
- else: # means no copyright line was found
- error(filename, 0, 'legal/copyright', 5,
- 'No copyright message found. '
- 'You should have a line: "Copyright [year] <Copyright Owner>"')
-
-
-def GetIndentLevel(line):
- """Return the number of leading spaces in line.
-
- Args:
- line: A string to check.
-
- Returns:
- An integer count of leading spaces, possibly zero.
- """
- indent = Match(r'^( *)\S', line)
- if indent:
- return len(indent.group(1))
- else:
- return 0
-
-
-def GetHeaderGuardCPPVariable(filename):
- """Returns the CPP variable that should be used as a header guard.
-
- Args:
- filename: The name of a C++ header file.
-
- Returns:
- The CPP variable that should be used as a header guard in the
- named file.
-
- """
-
- # Restores original filename in case that cpplint is invoked from Emacs's
- # flymake.
- filename = re.sub(r'_flymake\.h$', '.h', filename)
- filename = re.sub(r'/\.flymake/([^/]*)$', r'/\1', filename)
- # Replace 'c++' with 'cpp'.
- filename = filename.replace('C++', 'cpp').replace('c++', 'cpp')
-
- fileinfo = FileInfo(filename)
- file_path_from_root = fileinfo.RepositoryName()
- if _root:
- file_path_from_root = re.sub('^' + _root + os.sep, '', file_path_from_root)
- return re.sub(r'[^a-zA-Z0-9]', '_', file_path_from_root).upper() + '_'
-
-
-def CheckForHeaderGuard(filename, clean_lines, error):
- """Checks that the file contains a header guard.
-
- Logs an error if no #ifndef header guard is present. For other
- headers, checks that the full pathname is used.
-
- Args:
- filename: The name of the C++ header file.
- clean_lines: A CleansedLines instance containing the file.
- error: The function to call with any errors found.
- """
-
- # Don't check for header guards if there are error suppression
- # comments somewhere in this file.
- #
- # Because this is silencing a warning for a nonexistent line, we
- # only support the very specific NOLINT(build/header_guard) syntax,
- # and not the general NOLINT or NOLINT(*) syntax.
- raw_lines = clean_lines.lines_without_raw_strings
- for i in raw_lines:
- if Search(r'//\s*NOLINT\(build/header_guard\)', i):
- return
-
- cppvar = GetHeaderGuardCPPVariable(filename)
-
- ifndef = ''
- ifndef_linenum = 0
- define = ''
- endif = ''
- endif_linenum = 0
- for linenum, line in enumerate(raw_lines):
- linesplit = line.split()
- if len(linesplit) >= 2:
- # find the first occurrence of #ifndef and #define, save arg
- if not ifndef and linesplit[0] == '#ifndef':
- # set ifndef to the header guard presented on the #ifndef line.
- ifndef = linesplit[1]
- ifndef_linenum = linenum
- if not define and linesplit[0] == '#define':
- define = linesplit[1]
- # find the last occurrence of #endif, save entire line
- if line.startswith('#endif'):
- endif = line
- endif_linenum = linenum
-
- if not ifndef or not define or ifndef != define:
- error(filename, 0, 'build/header_guard', 5,
- 'No #ifndef header guard found, suggested CPP variable is: %s' %
- cppvar)
- return
-
- # The guard should be PATH_FILE_H_, but we also allow PATH_FILE_H__
- # for backward compatibility.
- if ifndef != cppvar:
- error_level = 0
- if ifndef != cppvar + '_':
- error_level = 5
-
- ParseNolintSuppressions(filename, raw_lines[ifndef_linenum], ifndef_linenum,
- error)
- error(filename, ifndef_linenum, 'build/header_guard', error_level,
- '#ifndef header guard has wrong style, please use: %s' % cppvar)
-
- # Check for "//" comments on endif line.
- ParseNolintSuppressions(filename, raw_lines[endif_linenum], endif_linenum,
- error)
- match = Match(r'#endif\s*//\s*' + cppvar + r'(_)?\b', endif)
- if match:
- if match.group(1) == '_':
- # Issue low severity warning for deprecated double trailing underscore
- error(filename, endif_linenum, 'build/header_guard', 0,
- '#endif line should be "#endif // %s"' % cppvar)
- return
-
- # Didn't find the corresponding "//" comment. If this file does not
- # contain any "//" comments at all, it could be that the compiler
- # only wants "/**/" comments, look for those instead.
- no_single_line_comments = True
- for i in xrange(1, len(raw_lines) - 1):
- line = raw_lines[i]
- if Match(r'^(?:(?:\'(?:\.|[^\'])*\')|(?:"(?:\.|[^"])*")|[^\'"])*//', line):
- no_single_line_comments = False
- break
-
- if no_single_line_comments:
- match = Match(r'#endif\s*/\*\s*' + cppvar + r'(_)?\s*\*/', endif)
- if match:
- if match.group(1) == '_':
- # Low severity warning for double trailing underscore
- error(filename, endif_linenum, 'build/header_guard', 0,
- '#endif line should be "#endif /* %s */"' % cppvar)
- return
-
- # Didn't find anything
- error(filename, endif_linenum, 'build/header_guard', 5,
- '#endif line should be "#endif // %s"' % cppvar)
-
-
-def CheckHeaderFileIncluded(filename, include_state, error):
- """Logs an error if a .cc file does not include its header."""
-
- # Do not check test files
- if filename.endswith('_test.cc') or filename.endswith('_unittest.cc'):
- return
-
- fileinfo = FileInfo(filename)
- headerfile = filename[0:len(filename) - 2] + 'h'
- if not os.path.exists(headerfile):
- return
- headername = FileInfo(headerfile).RepositoryName()
- first_include = 0
- for section_list in include_state.include_list:
- for f in section_list:
- if headername in f[0] or f[0] in headername:
- return
- if not first_include:
- first_include = f[1]
-
- error(filename, first_include, 'build/include', 5,
- '%s should include its header file %s' % (fileinfo.RepositoryName(),
- headername))
-
-
-def CheckForBadCharacters(filename, lines, error):
- """Logs an error for each line containing bad characters.
-
- Two kinds of bad characters:
-
- 1. Unicode replacement characters: These indicate that either the file
- contained invalid UTF-8 (likely) or Unicode replacement characters (which
- it shouldn't). Note that it's possible for this to throw off line
- numbering if the invalid UTF-8 occurred adjacent to a newline.
-
- 2. NUL bytes. These are problematic for some tools.
-
- Args:
- filename: The name of the current file.
- lines: An array of strings, each representing a line of the file.
- error: The function to call with any errors found.
- """
- for linenum, line in enumerate(lines):
- if u'\ufffd' in line:
- error(filename, linenum, 'readability/utf8', 5,
- 'Line contains invalid UTF-8 (or Unicode replacement character).')
- if '\0' in line:
- error(filename, linenum, 'readability/nul', 5, 'Line contains NUL byte.')
-
-
-def CheckForNewlineAtEOF(filename, lines, error):
- """Logs an error if there is no newline char at the end of the file.
-
- Args:
- filename: The name of the current file.
- lines: An array of strings, each representing a line of the file.
- error: The function to call with any errors found.
- """
-
- # The array lines() was created by adding two newlines to the
- # original file (go figure), then splitting on \n.
- # To verify that the file ends in \n, we just have to make sure the
- # last-but-two element of lines() exists and is empty.
- if len(lines) < 3 or lines[-2]:
- error(filename, len(lines) - 2, 'whitespace/ending_newline', 5,
- 'Could not find a newline character at the end of the file.')
-
-
-def CheckForMultilineCommentsAndStrings(filename, clean_lines, linenum, error):
- """Logs an error if we see /* ... */ or "..." that extend past one line.
-
- /* ... */ comments are legit inside macros, for one line.
- Otherwise, we prefer // comments, so it's ok to warn about the
- other. Likewise, it's ok for strings to extend across multiple
- lines, as long as a line continuation character (backslash)
- terminates each line. Although not currently prohibited by the C++
- style guide, it's ugly and unnecessary. We don't do well with either
- in this lint program, so we warn about both.
-
- Args:
- filename: The name of the current file.
- clean_lines: A CleansedLines instance containing the file.
- linenum: The number of the line to check.
- error: The function to call with any errors found.
- """
- line = clean_lines.elided[linenum]
-
- # Remove all \\ (escaped backslashes) from the line. They are OK, and the
- # second (escaped) slash may trigger later \" detection erroneously.
- line = line.replace('\\\\', '')
-
- if line.count('/*') > line.count('*/'):
- error(filename, linenum, 'readability/multiline_comment', 5,
- 'Complex multi-line /*...*/-style comment found. '
- 'Lint may give bogus warnings. '
- 'Consider replacing these with //-style comments, '
- 'with #if 0...#endif, '
- 'or with more clearly structured multi-line comments.')
-
- if (line.count('"') - line.count('\\"')) % 2:
- error(filename, linenum, 'readability/multiline_string', 5,
- 'Multi-line string ("...") found. This lint script doesn\'t '
- 'do well with such strings, and may give bogus warnings. '
- 'Use C++11 raw strings or concatenation instead.')
-
-
-# (non-threadsafe name, thread-safe alternative, validation pattern)
-#
-# The validation pattern is used to eliminate false positives such as:
-# _rand(); // false positive due to substring match.
-# ->rand(); // some member function rand().
-# ACMRandom rand(seed); // some variable named rand.
-# ISAACRandom rand(); // another variable named rand.
-#
-# Basically we require the return value of these functions to be used
-# in some expression context on the same line by matching on some
-# operator before the function name. This eliminates constructors and
-# member function calls.
-_UNSAFE_FUNC_PREFIX = r'(?:[-+*/=%^&|(<]\s*|>\s+)'
-_THREADING_LIST = (
- ('asctime(', 'asctime_r(', _UNSAFE_FUNC_PREFIX + r'asctime\([^)]+\)'),
- ('ctime(', 'ctime_r(', _UNSAFE_FUNC_PREFIX + r'ctime\([^)]+\)'),
- ('getgrgid(', 'getgrgid_r(', _UNSAFE_FUNC_PREFIX + r'getgrgid\([^)]+\)'),
- ('getgrnam(', 'getgrnam_r(', _UNSAFE_FUNC_PREFIX + r'getgrnam\([^)]+\)'),
- ('getlogin(', 'getlogin_r(', _UNSAFE_FUNC_PREFIX + r'getlogin\(\)'),
- ('getpwnam(', 'getpwnam_r(', _UNSAFE_FUNC_PREFIX + r'getpwnam\([^)]+\)'),
- ('getpwuid(', 'getpwuid_r(', _UNSAFE_FUNC_PREFIX + r'getpwuid\([^)]+\)'),
- ('gmtime(', 'gmtime_r(', _UNSAFE_FUNC_PREFIX + r'gmtime\([^)]+\)'),
- ('localtime(', 'localtime_r(', _UNSAFE_FUNC_PREFIX + r'localtime\([^)]+\)'),
- ('rand(', 'rand_r(', _UNSAFE_FUNC_PREFIX + r'rand\(\)'),
- ('strtok(', 'strtok_r(',
- _UNSAFE_FUNC_PREFIX + r'strtok\([^)]+\)'),
- ('ttyname(', 'ttyname_r(', _UNSAFE_FUNC_PREFIX + r'ttyname\([^)]+\)'),
- )
-
-
-def CheckPosixThreading(filename, clean_lines, linenum, error):
- """Checks for calls to thread-unsafe functions.
-
- Much code has been originally written without consideration of
- multi-threading. Also, engineers are relying on their old experience;
- they have learned posix before threading extensions were added. These
- tests guide the engineers to use thread-safe functions (when using
- posix directly).
-
- Args:
- filename: The name of the current file.
- clean_lines: A CleansedLines instance containing the file.
- linenum: The number of the line to check.
- error: The function to call with any errors found.
- """
- line = clean_lines.elided[linenum]
- for single_thread_func, multithread_safe_func, pattern in _THREADING_LIST:
- # Additional pattern matching check to confirm that this is the
- # function we are looking for
- if Search(pattern, line):
- error(filename, linenum, 'runtime/threadsafe_fn', 2,
- 'Consider using ' + multithread_safe_func +
- '...) instead of ' + single_thread_func +
- '...) for improved thread safety.')
-
-
-def CheckVlogArguments(filename, clean_lines, linenum, error):
- """Checks that VLOG() is only used for defining a logging level.
-
- For example, VLOG(2) is correct. VLOG(INFO), VLOG(WARNING), VLOG(ERROR), and
- VLOG(FATAL) are not.
-
- Args:
- filename: The name of the current file.
- clean_lines: A CleansedLines instance containing the file.
- linenum: The number of the line to check.
- error: The function to call with any errors found.
- """
- line = clean_lines.elided[linenum]
- if Search(r'\bVLOG\((INFO|ERROR|WARNING|DFATAL|FATAL)\)', line):
- error(filename, linenum, 'runtime/vlog', 5,
- 'VLOG() should be used with numeric verbosity level. '
- 'Use LOG() if you want symbolic severity levels.')
-
-# Matches invalid increment: *count++, which moves pointer instead of
-# incrementing a value.
-_RE_PATTERN_INVALID_INCREMENT = re.compile(
- r'^\s*\*\w+(\+\+|--);')
-
-
-def CheckInvalidIncrement(filename, clean_lines, linenum, error):
- """Checks for invalid increment *count++.
-
- For example following function:
- void increment_counter(int* count) {
- *count++;
- }
- is invalid, because it effectively does count++, moving pointer, and should
- be replaced with ++*count, (*count)++ or *count += 1.
-
- Args:
- filename: The name of the current file.
- clean_lines: A CleansedLines instance containing the file.
- linenum: The number of the line to check.
- error: The function to call with any errors found.
- """
- line = clean_lines.elided[linenum]
- if _RE_PATTERN_INVALID_INCREMENT.match(line):
- error(filename, linenum, 'runtime/invalid_increment', 5,
- 'Changing pointer instead of value (or unused value of operator*).')
-
-
-def IsMacroDefinition(clean_lines, linenum):
- if Search(r'^#define', clean_lines[linenum]):
- return True
-
- if linenum > 0 and Search(r'\\$', clean_lines[linenum - 1]):
- return True
-
- return False
-
-
-def IsForwardClassDeclaration(clean_lines, linenum):
- return Match(r'^\s*(\btemplate\b)*.*class\s+\w+;\s*$', clean_lines[linenum])
-
-
-class _BlockInfo(object):
- """Stores information about a generic block of code."""
-
- def __init__(self, seen_open_brace):
- self.seen_open_brace = seen_open_brace
- self.open_parentheses = 0
- self.inline_asm = _NO_ASM
- self.check_namespace_indentation = False
-
- def CheckBegin(self, filename, clean_lines, linenum, error):
- """Run checks that applies to text up to the opening brace.
-
- This is mostly for checking the text after the class identifier
- and the "{", usually where the base class is specified. For other
- blocks, there isn't much to check, so we always pass.
-
- Args:
- filename: The name of the current file.
- clean_lines: A CleansedLines instance containing the file.
- linenum: The number of the line to check.
- error: The function to call with any errors found.
- """
- pass
-
- def CheckEnd(self, filename, clean_lines, linenum, error):
- """Run checks that applies to text after the closing brace.
-
- This is mostly used for checking end of namespace comments.
-
- Args:
- filename: The name of the current file.
- clean_lines: A CleansedLines instance containing the file.
- linenum: The number of the line to check.
- error: The function to call with any errors found.
- """
- pass
-
- def IsBlockInfo(self):
- """Returns true if this block is a _BlockInfo.
-
- This is convenient for verifying that an object is an instance of
- a _BlockInfo, but not an instance of any of the derived classes.
-
- Returns:
- True for this class, False for derived classes.
- """
- return self.__class__ == _BlockInfo
-
-
-class _ExternCInfo(_BlockInfo):
- """Stores information about an 'extern "C"' block."""
-
- def __init__(self):
- _BlockInfo.__init__(self, True)
-
-
-class _ClassInfo(_BlockInfo):
- """Stores information about a class."""
-
- def __init__(self, name, class_or_struct, clean_lines, linenum):
- _BlockInfo.__init__(self, False)
- self.name = name
- self.starting_linenum = linenum
- self.is_derived = False
- self.check_namespace_indentation = True
- if class_or_struct == 'struct':
- self.access = 'public'
- self.is_struct = True
- else:
- self.access = 'private'
- self.is_struct = False
-
- # Remember initial indentation level for this class. Using raw_lines here
- # instead of elided to account for leading comments.
- self.class_indent = GetIndentLevel(clean_lines.raw_lines[linenum])
-
- # Try to find the end of the class. This will be confused by things like:
- # class A {
- # } *x = { ...
- #
- # But it's still good enough for CheckSectionSpacing.
- self.last_line = 0
- depth = 0
- for i in range(linenum, clean_lines.NumLines()):
- line = clean_lines.elided[i]
- depth += line.count('{') - line.count('}')
- if not depth:
- self.last_line = i
- break
-
- def CheckBegin(self, filename, clean_lines, linenum, error):
- # Look for a bare ':'
- if Search('(^|[^:]):($|[^:])', clean_lines.elided[linenum]):
- self.is_derived = True
-
- def CheckEnd(self, filename, clean_lines, linenum, error):
- # If there is a DISALLOW macro, it should appear near the end of
- # the class.
- seen_last_thing_in_class = False
- for i in xrange(linenum - 1, self.starting_linenum, -1):
- match = Search(
- r'\b(DISALLOW_COPY_AND_ASSIGN|DISALLOW_IMPLICIT_CONSTRUCTORS)\(' +
- self.name + r'\)',
- clean_lines.elided[i])
- if match:
- if seen_last_thing_in_class:
- error(filename, i, 'readability/constructors', 3,
- match.group(1) + ' should be the last thing in the class')
- break
-
- if not Match(r'^\s*$', clean_lines.elided[i]):
- seen_last_thing_in_class = True
-
- # Check that closing brace is aligned with beginning of the class.
- # Only do this if the closing brace is indented by only whitespaces.
- # This means we will not check single-line class definitions.
- indent = Match(r'^( *)\}', clean_lines.elided[linenum])
- if indent and len(indent.group(1)) != self.class_indent:
- if self.is_struct:
- parent = 'struct ' + self.name
- else:
- parent = 'class ' + self.name
- error(filename, linenum, 'whitespace/indent', 3,
- 'Closing brace should be aligned with beginning of %s' % parent)
-
-
-class _NamespaceInfo(_BlockInfo):
- """Stores information about a namespace."""
-
- def __init__(self, name, linenum):
- _BlockInfo.__init__(self, False)
- self.name = name or ''
- self.starting_linenum = linenum
- self.check_namespace_indentation = True
-
- def CheckEnd(self, filename, clean_lines, linenum, error):
- """Check end of namespace comments."""
- line = clean_lines.raw_lines[linenum]
-
- # Check how many lines is enclosed in this namespace. Don't issue
- # warning for missing namespace comments if there aren't enough
- # lines. However, do apply checks if there is already an end of
- # namespace comment and it's incorrect.
- #
- # TODO(unknown): We always want to check end of namespace comments
- # if a namespace is large, but sometimes we also want to apply the
- # check if a short namespace contained nontrivial things (something
- # other than forward declarations). There is currently no logic on
- # deciding what these nontrivial things are, so this check is
- # triggered by namespace size only, which works most of the time.
- if (linenum - self.starting_linenum < 10
- and not Match(r'};*\s*(//|/\*).*\bnamespace\b', line)):
- return
-
- # Look for matching comment at end of namespace.
- #
- # Note that we accept C style "/* */" comments for terminating
- # namespaces, so that code that terminate namespaces inside
- # preprocessor macros can be cpplint clean.
- #
- # We also accept stuff like "// end of namespace <name>." with the
- # period at the end.
- #
- # Besides these, we don't accept anything else, otherwise we might
- # get false negatives when existing comment is a substring of the
- # expected namespace.
- if self.name:
- # Named namespace
- if not Match((r'};*\s*(//|/\*).*\bnamespace\s+' + re.escape(self.name) +
- r'[\*/\.\\\s]*$'),
- line):
- error(filename, linenum, 'readability/namespace', 5,
- 'Namespace should be terminated with "// namespace %s"' %
- self.name)
- else:
- # Anonymous namespace
- if not Match(r'};*\s*(//|/\*).*\bnamespace[\*/\.\\\s]*$', line):
- # If "// namespace anonymous" or "// anonymous namespace (more text)",
- # mention "// anonymous namespace" as an acceptable form
- if Match(r'}.*\b(namespace anonymous|anonymous namespace)\b', line):
- error(filename, linenum, 'readability/namespace', 5,
- 'Anonymous namespace should be terminated with "// namespace"'
- ' or "// anonymous namespace"')
- else:
- error(filename, linenum, 'readability/namespace', 5,
- 'Anonymous namespace should be terminated with "// namespace"')
-
-
-class _PreprocessorInfo(object):
- """Stores checkpoints of nesting stacks when #if/#else is seen."""
-
- def __init__(self, stack_before_if):
- # The entire nesting stack before #if
- self.stack_before_if = stack_before_if
-
- # The entire nesting stack up to #else
- self.stack_before_else = []
-
- # Whether we have already seen #else or #elif
- self.seen_else = False
-
-
-class NestingState(object):
- """Holds states related to parsing braces."""
-
- def __init__(self):
- # Stack for tracking all braces. An object is pushed whenever we
- # see a "{", and popped when we see a "}". Only 3 types of
- # objects are possible:
- # - _ClassInfo: a class or struct.
- # - _NamespaceInfo: a namespace.
- # - _BlockInfo: some other type of block.
- self.stack = []
-
- # Top of the previous stack before each Update().
- #
- # Because the nesting_stack is updated at the end of each line, we
- # had to do some convoluted checks to find out what is the current
- # scope at the beginning of the line. This check is simplified by
- # saving the previous top of nesting stack.
- #
- # We could save the full stack, but we only need the top. Copying
- # the full nesting stack would slow down cpplint by ~10%.
- self.previous_stack_top = []
-
- # Stack of _PreprocessorInfo objects.
- self.pp_stack = []
-
- def SeenOpenBrace(self):
- """Check if we have seen the opening brace for the innermost block.
-
- Returns:
- True if we have seen the opening brace, False if the innermost
- block is still expecting an opening brace.
- """
- return (not self.stack) or self.stack[-1].seen_open_brace
-
- def InNamespaceBody(self):
- """Check if we are currently one level inside a namespace body.
-
- Returns:
- True if top of the stack is a namespace block, False otherwise.
- """
- return self.stack and isinstance(self.stack[-1], _NamespaceInfo)
-
- def InExternC(self):
- """Check if we are currently one level inside an 'extern "C"' block.
-
- Returns:
- True if top of the stack is an extern block, False otherwise.
- """
- return self.stack and isinstance(self.stack[-1], _ExternCInfo)
-
- def InClassDeclaration(self):
- """Check if we are currently one level inside a class or struct declaration.
-
- Returns:
- True if top of the stack is a class/struct, False otherwise.
- """
- return self.stack and isinstance(self.stack[-1], _ClassInfo)
-
- def InAsmBlock(self):
- """Check if we are currently one level inside an inline ASM block.
-
- Returns:
- True if the top of the stack is a block containing inline ASM.
- """
- return self.stack and self.stack[-1].inline_asm != _NO_ASM
-
- def InTemplateArgumentList(self, clean_lines, linenum, pos):
- """Check if current position is inside template argument list.
-
- Args:
- clean_lines: A CleansedLines instance containing the file.
- linenum: The number of the line to check.
- pos: position just after the suspected template argument.
- Returns:
- True if (linenum, pos) is inside template arguments.
- """
- while linenum < clean_lines.NumLines():
- # Find the earliest character that might indicate a template argument
- line = clean_lines.elided[linenum]
- match = Match(r'^[^{};=\[\]\.<>]*(.)', line[pos:])
- if not match:
- linenum += 1
- pos = 0
- continue
- token = match.group(1)
- pos += len(match.group(0))
-
- # These things do not look like template argument list:
- # class Suspect {
- # class Suspect x; }
- if token in ('{', '}', ';'): return False
-
- # These things look like template argument list:
- # template <class Suspect>
- # template <class Suspect = default_value>
- # template <class Suspect[]>
- # template <class Suspect...>
- if token in ('>', '=', '[', ']', '.'): return True
-
- # Check if token is an unmatched '<'.
- # If not, move on to the next character.
- if token != '<':
- pos += 1
- if pos >= len(line):
- linenum += 1
- pos = 0
- continue
-
- # We can't be sure if we just find a single '<', and need to
- # find the matching '>'.
- (_, end_line, end_pos) = CloseExpression(clean_lines, linenum, pos - 1)
- if end_pos < 0:
- # Not sure if template argument list or syntax error in file
- return False
- linenum = end_line
- pos = end_pos
- return False
-
- def UpdatePreprocessor(self, line):
- """Update preprocessor stack.
-
- We need to handle preprocessors due to classes like this:
- #ifdef SWIG
- struct ResultDetailsPageElementExtensionPoint {
- #else
- struct ResultDetailsPageElementExtensionPoint : public Extension {
- #endif
-
- We make the following assumptions (good enough for most files):
- - Preprocessor condition evaluates to true from #if up to first
- #else/#elif/#endif.
-
- - Preprocessor condition evaluates to false from #else/#elif up
- to #endif. We still perform lint checks on these lines, but
- these do not affect nesting stack.
-
- Args:
- line: current line to check.
- """
- if Match(r'^\s*#\s*(if|ifdef|ifndef)\b', line):
- # Beginning of #if block, save the nesting stack here. The saved
- # stack will allow us to restore the parsing state in the #else case.
- self.pp_stack.append(_PreprocessorInfo(copy.deepcopy(self.stack)))
- elif Match(r'^\s*#\s*(else|elif)\b', line):
- # Beginning of #else block
- if self.pp_stack:
- if not self.pp_stack[-1].seen_else:
- # This is the first #else or #elif block. Remember the
- # whole nesting stack up to this point. This is what we
- # keep after the #endif.
- self.pp_stack[-1].seen_else = True
- self.pp_stack[-1].stack_before_else = copy.deepcopy(self.stack)
-
- # Restore the stack to how it was before the #if
- self.stack = copy.deepcopy(self.pp_stack[-1].stack_before_if)
- else:
- # TODO(unknown): unexpected #else, issue warning?
- pass
- elif Match(r'^\s*#\s*endif\b', line):
- # End of #if or #else blocks.
- if self.pp_stack:
- # If we saw an #else, we will need to restore the nesting
- # stack to its former state before the #else, otherwise we
- # will just continue from where we left off.
- if self.pp_stack[-1].seen_else:
- # Here we can just use a shallow copy since we are the last
- # reference to it.
- self.stack = self.pp_stack[-1].stack_before_else
- # Drop the corresponding #if
- self.pp_stack.pop()
- else:
- # TODO(unknown): unexpected #endif, issue warning?
- pass
-
- # TODO(unknown): Update() is too long, but we will refactor later.
- def Update(self, filename, clean_lines, linenum, error):
- """Update nesting state with current line.
-
- Args:
- filename: The name of the current file.
- clean_lines: A CleansedLines instance containing the file.
- linenum: The number of the line to check.
- error: The function to call with any errors found.
- """
- line = clean_lines.elided[linenum]
-
- # Remember top of the previous nesting stack.
- #
- # The stack is always pushed/popped and not modified in place, so
- # we can just do a shallow copy instead of copy.deepcopy. Using
- # deepcopy would slow down cpplint by ~28%.
- if self.stack:
- self.previous_stack_top = self.stack[-1]
- else:
- self.previous_stack_top = None
-
- # Update pp_stack
- self.UpdatePreprocessor(line)
-
- # Count parentheses. This is to avoid adding struct arguments to
- # the nesting stack.
- if self.stack:
- inner_block = self.stack[-1]
- depth_change = line.count('(') - line.count(')')
- inner_block.open_parentheses += depth_change
-
- # Also check if we are starting or ending an inline assembly block.
- if inner_block.inline_asm in (_NO_ASM, _END_ASM):
- if (depth_change != 0 and
- inner_block.open_parentheses == 1 and
- _MATCH_ASM.match(line)):
- # Enter assembly block
- inner_block.inline_asm = _INSIDE_ASM
- else:
- # Not entering assembly block. If previous line was _END_ASM,
- # we will now shift to _NO_ASM state.
- inner_block.inline_asm = _NO_ASM
- elif (inner_block.inline_asm == _INSIDE_ASM and
- inner_block.open_parentheses == 0):
- # Exit assembly block
- inner_block.inline_asm = _END_ASM
-
- # Consume namespace declaration at the beginning of the line. Do
- # this in a loop so that we catch same line declarations like this:
- # namespace proto2 { namespace bridge { class MessageSet; } }
- while True:
- # Match start of namespace. The "\b\s*" below catches namespace
- # declarations even if it weren't followed by a whitespace, this
- # is so that we don't confuse our namespace checker. The
- # missing spaces will be flagged by CheckSpacing.
- namespace_decl_match = Match(r'^\s*namespace\b\s*([:\w]+)?(.*)$', line)
- if not namespace_decl_match:
- break
-
- new_namespace = _NamespaceInfo(namespace_decl_match.group(1), linenum)
- self.stack.append(new_namespace)
-
- line = namespace_decl_match.group(2)
- if line.find('{') != -1:
- new_namespace.seen_open_brace = True
- line = line[line.find('{') + 1:]
-
- # Look for a class declaration in whatever is left of the line
- # after parsing namespaces. The regexp accounts for decorated classes
- # such as in:
- # class LOCKABLE API Object {
- # };
- class_decl_match = Match(
- r'^(\s*(?:template\s*<[\w\s<>,:]*>\s*)?'
- r'(class|struct)\s+(?:[A-Z_]+\s+)*(\w+(?:::\w+)*))'
- r'(.*)$', line)
- if (class_decl_match and
- (not self.stack or self.stack[-1].open_parentheses == 0)):
- # We do not want to accept classes that are actually template arguments:
- # template <class Ignore1,
- # class Ignore2 = Default<Args>,
- # template <Args> class Ignore3>
- # void Function() {};
- #
- # To avoid template argument cases, we scan forward and look for
- # an unmatched '>'. If we see one, assume we are inside a
- # template argument list.
- end_declaration = len(class_decl_match.group(1))
- if not self.InTemplateArgumentList(clean_lines, linenum, end_declaration):
- self.stack.append(_ClassInfo(
- class_decl_match.group(3), class_decl_match.group(2),
- clean_lines, linenum))
- line = class_decl_match.group(4)
-
- # If we have not yet seen the opening brace for the innermost block,
- # run checks here.
- if not self.SeenOpenBrace():
- self.stack[-1].CheckBegin(filename, clean_lines, linenum, error)
-
- # Update access control if we are inside a class/struct
- if self.stack and isinstance(self.stack[-1], _ClassInfo):
- classinfo = self.stack[-1]
- access_match = Match(
- r'^(.*)\b(public|private|protected|signals)(\s+(?:slots\s*)?)?'
- r':(?:[^:]|$)',
- line)
- if access_match:
- classinfo.access = access_match.group(2)
-
- # Check that access keywords are indented +1 space. Skip this
- # check if the keywords are not preceded by whitespaces.
- indent = access_match.group(1)
- if (len(indent) != classinfo.class_indent + 1 and
- Match(r'^\s*$', indent)):
- if classinfo.is_struct:
- parent = 'struct ' + classinfo.name
- else:
- parent = 'class ' + classinfo.name
- slots = ''
- if access_match.group(3):
- slots = access_match.group(3)
- error(filename, linenum, 'whitespace/indent', 3,
- '%s%s: should be indented +1 space inside %s' % (
- access_match.group(2), slots, parent))
-
- # Consume braces or semicolons from what's left of the line
- while True:
- # Match first brace, semicolon, or closed parenthesis.
- matched = Match(r'^[^{;)}]*([{;)}])(.*)$', line)
- if not matched:
- break
-
- token = matched.group(1)
- if token == '{':
- # If namespace or class hasn't seen a opening brace yet, mark
- # namespace/class head as complete. Push a new block onto the
- # stack otherwise.
- if not self.SeenOpenBrace():
- self.stack[-1].seen_open_brace = True
- elif Match(r'^extern\s*"[^"]*"\s*\{', line):
- self.stack.append(_ExternCInfo())
- else:
- self.stack.append(_BlockInfo(True))
- if _MATCH_ASM.match(line):
- self.stack[-1].inline_asm = _BLOCK_ASM
-
- elif token == ';' or token == ')':
- # If we haven't seen an opening brace yet, but we already saw
- # a semicolon, this is probably a forward declaration. Pop
- # the stack for these.
- #
- # Similarly, if we haven't seen an opening brace yet, but we
- # already saw a closing parenthesis, then these are probably
- # function arguments with extra "class" or "struct" keywords.
- # Also pop these stack for these.
- if not self.SeenOpenBrace():
- self.stack.pop()
- else: # token == '}'
- # Perform end of block checks and pop the stack.
- if self.stack:
- self.stack[-1].CheckEnd(filename, clean_lines, linenum, error)
- self.stack.pop()
- line = matched.group(2)
-
- def InnermostClass(self):
- """Get class info on the top of the stack.
-
- Returns:
- A _ClassInfo object if we are inside a class, or None otherwise.
- """
- for i in range(len(self.stack), 0, -1):
- classinfo = self.stack[i - 1]
- if isinstance(classinfo, _ClassInfo):
- return classinfo
- return None
-
- def CheckCompletedBlocks(self, filename, error):
- """Checks that all classes and namespaces have been completely parsed.
-
- Call this when all lines in a file have been processed.
- Args:
- filename: The name of the current file.
- error: The function to call with any errors found.
- """
- # Note: This test can result in false positives if #ifdef constructs
- # get in the way of brace matching. See the testBuildClass test in
- # cpplint_unittest.py for an example of this.
- for obj in self.stack:
- if isinstance(obj, _ClassInfo):
- error(filename, obj.starting_linenum, 'build/class', 5,
- 'Failed to find complete declaration of class %s' %
- obj.name)
- elif isinstance(obj, _NamespaceInfo):
- error(filename, obj.starting_linenum, 'build/namespaces', 5,
- 'Failed to find complete declaration of namespace %s' %
- obj.name)
-
-
-def CheckForNonStandardConstructs(filename, clean_lines, linenum,
- nesting_state, error):
- r"""Logs an error if we see certain non-ANSI constructs ignored by gcc-2.
-
- Complain about several constructs which gcc-2 accepts, but which are
- not standard C++. Warning about these in lint is one way to ease the
- transition to new compilers.
- - put storage class first (e.g. "static const" instead of "const static").
- - "%lld" instead of %qd" in printf-type functions.
- - "%1$d" is non-standard in printf-type functions.
- - "\%" is an undefined character escape sequence.
- - text after #endif is not allowed.
- - invalid inner-style forward declaration.
- - >? and <? operators, and their >?= and <?= cousins.
-
- Additionally, check for constructor/destructor style violations and reference
- members, as it is very convenient to do so while checking for
- gcc-2 compliance.
-
- Args:
- filename: The name of the current file.
- clean_lines: A CleansedLines instance containing the file.
- linenum: The number of the line to check.
- nesting_state: A NestingState instance which maintains information about
- the current stack of nested blocks being parsed.
- error: A callable to which errors are reported, which takes 4 arguments:
- filename, line number, error level, and message
- """
-
- # Remove comments from the line, but leave in strings for now.
- line = clean_lines.lines[linenum]
-
- if Search(r'printf\s*\(.*".*%[-+ ]?\d*q', line):
- error(filename, linenum, 'runtime/printf_format', 3,
- '%q in format strings is deprecated. Use %ll instead.')
-
- if Search(r'printf\s*\(.*".*%\d+\$', line):
- error(filename, linenum, 'runtime/printf_format', 2,
- '%N$ formats are unconventional. Try rewriting to avoid them.')
-
- # Remove escaped backslashes before looking for undefined escapes.
- line = line.replace('\\\\', '')
-
- if Search(r'("|\').*\\(%|\[|\(|{)', line):
- error(filename, linenum, 'build/printf_format', 3,
- '%, [, (, and { are undefined character escapes. Unescape them.')
-
- # For the rest, work with both comments and strings removed.
- line = clean_lines.elided[linenum]
-
- if Search(r'\b(const|volatile|void|char|short|int|long'
- r'|float|double|signed|unsigned'
- r'|schar|u?int8|u?int16|u?int32|u?int64)'
- r'\s+(register|static|extern|typedef)\b',
- line):
- error(filename, linenum, 'build/storage_class', 5,
- 'Storage class (static, extern, typedef, etc) should be first.')
-
- if Match(r'\s*#\s*endif\s*[^/\s]+', line):
- error(filename, linenum, 'build/endif_comment', 5,
- 'Uncommented text after #endif is non-standard. Use a comment.')
-
- if Match(r'\s*class\s+(\w+\s*::\s*)+\w+\s*;', line):
- error(filename, linenum, 'build/forward_decl', 5,
- 'Inner-style forward declarations are invalid. Remove this line.')
-
- if Search(r'(\w+|[+-]?\d+(\.\d*)?)\s*(<|>)\?=?\s*(\w+|[+-]?\d+)(\.\d*)?',
- line):
- error(filename, linenum, 'build/deprecated', 3,
- '>? and <? (max and min) operators are non-standard and deprecated.')
-
- if Search(r'^\s*const\s*string\s*&\s*\w+\s*;', line):
- # TODO(unknown): Could it be expanded safely to arbitrary references,
- # without triggering too many false positives? The first
- # attempt triggered 5 warnings for mostly benign code in the regtest, hence
- # the restriction.
- # Here's the original regexp, for the reference:
- # type_name = r'\w+((\s*::\s*\w+)|(\s*<\s*\w+?\s*>))?'
- # r'\s*const\s*' + type_name + '\s*&\s*\w+\s*;'
- error(filename, linenum, 'runtime/member_string_references', 2,
- 'const string& members are dangerous. It is much better to use '
- 'alternatives, such as pointers or simple constants.')
-
- # Everything else in this function operates on class declarations.
- # Return early if the top of the nesting stack is not a class, or if
- # the class head is not completed yet.
- classinfo = nesting_state.InnermostClass()
- if not classinfo or not classinfo.seen_open_brace:
- return
-
- # The class may have been declared with namespace or classname qualifiers.
- # The constructor and destructor will not have those qualifiers.
- base_classname = classinfo.name.split('::')[-1]
-
- # Look for single-argument constructors that aren't marked explicit.
- # Technically a valid construct, but against style. Also look for
- # non-single-argument constructors which are also technically valid, but
- # strongly suggest something is wrong.
- explicit_constructor_match = Match(
- r'\s+(?:inline\s+)?(explicit\s+)?(?:inline\s+)?%s\s*'
- r'\(((?:[^()]|\([^()]*\))*)\)'
- % re.escape(base_classname),
- line)
-
- if explicit_constructor_match:
- is_marked_explicit = explicit_constructor_match.group(1)
-
- if not explicit_constructor_match.group(2):
- constructor_args = []
- else:
- constructor_args = explicit_constructor_match.group(2).split(',')
-
- # collapse arguments so that commas in template parameter lists and function
- # argument parameter lists don't split arguments in two
- i = 0
- while i < len(constructor_args):
- constructor_arg = constructor_args[i]
- while (constructor_arg.count('<') > constructor_arg.count('>') or
- constructor_arg.count('(') > constructor_arg.count(')')):
- constructor_arg += ',' + constructor_args[i + 1]
- del constructor_args[i + 1]
- constructor_args[i] = constructor_arg
- i += 1
-
- defaulted_args = [arg for arg in constructor_args if '=' in arg]
- noarg_constructor = (not constructor_args or # empty arg list
- # 'void' arg specifier
- (len(constructor_args) == 1 and
- constructor_args[0].strip() == 'void'))
- onearg_constructor = ((len(constructor_args) == 1 and # exactly one arg
- not noarg_constructor) or
- # all but at most one arg defaulted
- (len(constructor_args) >= 1 and
- not noarg_constructor and
- len(defaulted_args) >= len(constructor_args) - 1))
- initializer_list_constructor = bool(
- onearg_constructor and
- Search(r'\bstd\s*::\s*initializer_list\b', constructor_args[0]))
- copy_constructor = bool(
- onearg_constructor and
- Match(r'(const\s+)?%s(\s*<[^>]*>)?(\s+const)?\s*(?:<\w+>\s*)?&'
- % re.escape(base_classname), constructor_args[0].strip()))
-
- if (not is_marked_explicit and
- onearg_constructor and
- not initializer_list_constructor and
- not copy_constructor):
- if defaulted_args:
- error(filename, linenum, 'runtime/explicit', 5,
- 'Constructors callable with one argument '
- 'should be marked explicit.')
- else:
- error(filename, linenum, 'runtime/explicit', 5,
- 'Single-parameter constructors should be marked explicit.')
- elif is_marked_explicit and not onearg_constructor:
- if noarg_constructor:
- error(filename, linenum, 'runtime/explicit', 5,
- 'Zero-parameter constructors should not be marked explicit.')
- else:
- error(filename, linenum, 'runtime/explicit', 0,
- 'Constructors that require multiple arguments '
- 'should not be marked explicit.')
-
-
-def CheckSpacingForFunctionCall(filename, clean_lines, linenum, error):
- """Checks for the correctness of various spacing around function calls.
-
- Args:
- filename: The name of the current file.
- clean_lines: A CleansedLines instance containing the file.
- linenum: The number of the line to check.
- error: The function to call with any errors found.
- """
- line = clean_lines.elided[linenum]
-
- # Since function calls often occur inside if/for/while/switch
- # expressions - which have their own, more liberal conventions - we
- # first see if we should be looking inside such an expression for a
- # function call, to which we can apply more strict standards.
- fncall = line # if there's no control flow construct, look at whole line
- for pattern in (r'\bif\s*\((.*)\)\s*{',
- r'\bfor\s*\((.*)\)\s*{',
- r'\bwhile\s*\((.*)\)\s*[{;]',
- r'\bswitch\s*\((.*)\)\s*{'):
- match = Search(pattern, line)
- if match:
- fncall = match.group(1) # look inside the parens for function calls
- break
-
- # Except in if/for/while/switch, there should never be space
- # immediately inside parens (eg "f( 3, 4 )"). We make an exception
- # for nested parens ( (a+b) + c ). Likewise, there should never be
- # a space before a ( when it's a function argument. I assume it's a
- # function argument when the char before the whitespace is legal in
- # a function name (alnum + _) and we're not starting a macro. Also ignore
- # pointers and references to arrays and functions coz they're too tricky:
- # we use a very simple way to recognize these:
- # " (something)(maybe-something)" or
- # " (something)(maybe-something," or
- # " (something)[something]"
- # Note that we assume the contents of [] to be short enough that
- # they'll never need to wrap.
- if ( # Ignore control structures.
- not Search(r'\b(if|for|while|switch|return|new|delete|catch|sizeof)\b',
- fncall) and
- # Ignore pointers/references to functions.
- not Search(r' \([^)]+\)\([^)]*(\)|,$)', fncall) and
- # Ignore pointers/references to arrays.
- not Search(r' \([^)]+\)\[[^\]]+\]', fncall)):
- if Search(r'\w\s*\(\s(?!\s*\\$)', fncall): # a ( used for a fn call
- error(filename, linenum, 'whitespace/parens', 4,
- 'Extra space after ( in function call')
- elif Search(r'\(\s+(?!(\s*\\)|\()', fncall):
- error(filename, linenum, 'whitespace/parens', 2,
- 'Extra space after (')
- if (Search(r'\w\s+\(', fncall) and
- not Search(r'#\s*define|typedef|using\s+\w+\s*=', fncall) and
- not Search(r'\w\s+\((\w+::)*\*\w+\)\(', fncall) and
- not Search(r'\bcase\s+\(', fncall)):
- # TODO(unknown): Space after an operator function seem to be a common
- # error, silence those for now by restricting them to highest verbosity.
- if Search(r'\boperator_*\b', line):
- error(filename, linenum, 'whitespace/parens', 0,
- 'Extra space before ( in function call')
- else:
- error(filename, linenum, 'whitespace/parens', 4,
- 'Extra space before ( in function call')
- # If the ) is followed only by a newline or a { + newline, assume it's
- # part of a control statement (if/while/etc), and don't complain
- if Search(r'[^)]\s+\)\s*[^{\s]', fncall):
- # If the closing parenthesis is preceded by only whitespaces,
- # try to give a more descriptive error message.
- if Search(r'^\s+\)', fncall):
- error(filename, linenum, 'whitespace/parens', 2,
- 'Closing ) should be moved to the previous line')
- else:
- error(filename, linenum, 'whitespace/parens', 2,
- 'Extra space before )')
-
-
-def IsBlankLine(line):
- """Returns true if the given line is blank.
-
- We consider a line to be blank if the line is empty or consists of
- only white spaces.
-
- Args:
- line: A line of a string.
-
- Returns:
- True, if the given line is blank.
- """
- return not line or line.isspace()
-
-
-def CheckForNamespaceIndentation(filename, nesting_state, clean_lines, line,
- error):
- is_namespace_indent_item = (
- len(nesting_state.stack) > 1 and
- nesting_state.stack[-1].check_namespace_indentation and
- isinstance(nesting_state.previous_stack_top, _NamespaceInfo) and
- nesting_state.previous_stack_top == nesting_state.stack[-2])
-
- if ShouldCheckNamespaceIndentation(nesting_state, is_namespace_indent_item,
- clean_lines.elided, line):
- CheckItemIndentationInNamespace(filename, clean_lines.elided,
- line, error)
-
-
-def CheckForFunctionLengths(filename, clean_lines, linenum,
- function_state, error):
- """Reports for long function bodies.
-
- For an overview why this is done, see:
- http://google-styleguide.googlecode.com/svn/trunk/cppguide.xml#Write_Short_Functions
-
- Uses a simplistic algorithm assuming other style guidelines
- (especially spacing) are followed.
- Only checks unindented functions, so class members are unchecked.
- Trivial bodies are unchecked, so constructors with huge initializer lists
- may be missed.
- Blank/comment lines are not counted so as to avoid encouraging the removal
- of vertical space and comments just to get through a lint check.
- NOLINT *on the last line of a function* disables this check.
-
- Args:
- filename: The name of the current file.
- clean_lines: A CleansedLines instance containing the file.
- linenum: The number of the line to check.
- function_state: Current function name and lines in body so far.
- error: The function to call with any errors found.
- """
- lines = clean_lines.lines
- line = lines[linenum]
- joined_line = ''
-
- starting_func = False
- regexp = r'(\w(\w|::|\*|\&|\s)*)\(' # decls * & space::name( ...
- match_result = Match(regexp, line)
- if match_result:
- # If the name is all caps and underscores, figure it's a macro and
- # ignore it, unless it's TEST or TEST_F.
- function_name = match_result.group(1).split()[-1]
- if function_name == 'TEST' or function_name == 'TEST_F' or (
- not Match(r'[A-Z_]+$', function_name)):
- starting_func = True
-
- if starting_func:
- body_found = False
- for start_linenum in xrange(linenum, clean_lines.NumLines()):
- start_line = lines[start_linenum]
- joined_line += ' ' + start_line.lstrip()
- if Search(r'(;|})', start_line): # Declarations and trivial functions
- body_found = True
- break # ... ignore
- elif Search(r'{', start_line):
- body_found = True
- function = Search(r'((\w|:)*)\(', line).group(1)
- if Match(r'TEST', function): # Handle TEST... macros
- parameter_regexp = Search(r'(\(.*\))', joined_line)
- if parameter_regexp: # Ignore bad syntax
- function += parameter_regexp.group(1)
- else:
- function += '()'
- function_state.Begin(function)
- break
- if not body_found:
- # No body for the function (or evidence of a non-function) was found.
- error(filename, linenum, 'readability/fn_size', 5,
- 'Lint failed to find start of function body.')
- elif Match(r'^\}\s*$', line): # function end
- function_state.Check(error, filename, linenum)
- function_state.End()
- elif not Match(r'^\s*$', line):
- function_state.Count() # Count non-blank/non-comment lines.
-
-
-_RE_PATTERN_TODO = re.compile(r'^//(\s*)TODO(\(.+?\))?:?(\s|$)?')
-
-
-def CheckComment(line, filename, linenum, next_line_start, error):
- """Checks for common mistakes in comments.
-
- Args:
- line: The line in question.
- filename: The name of the current file.
- linenum: The number of the line to check.
- next_line_start: The first non-whitespace column of the next line.
- error: The function to call with any errors found.
- """
- commentpos = line.find('//')
- if commentpos != -1:
- # Check if the // may be in quotes. If so, ignore it
- # Comparisons made explicit for clarity -- pylint: disable=g-explicit-bool-comparison
- if (line.count('"', 0, commentpos) -
- line.count('\\"', 0, commentpos)) % 2 == 0: # not in quotes
- # Allow one space for new scopes, two spaces otherwise:
- if (not (Match(r'^.*{ *//', line) and next_line_start == commentpos) and
- ((commentpos >= 1 and
- line[commentpos-1] not in string.whitespace) or
- (commentpos >= 2 and
- line[commentpos-2] not in string.whitespace))):
- error(filename, linenum, 'whitespace/comments', 2,
- 'At least two spaces is best between code and comments')
-
- # Checks for common mistakes in TODO comments.
- comment = line[commentpos:]
- match = _RE_PATTERN_TODO.match(comment)
- if match:
- # One whitespace is correct; zero whitespace is handled elsewhere.
- leading_whitespace = match.group(1)
- if len(leading_whitespace) > 1:
- error(filename, linenum, 'whitespace/todo', 2,
- 'Too many spaces before TODO')
-
- username = match.group(2)
- if not username:
- error(filename, linenum, 'readability/todo', 2,
- 'Missing username in TODO; it should look like '
- '"// TODO(my_username): Stuff."')
-
- middle_whitespace = match.group(3)
- # Comparisons made explicit for correctness -- pylint: disable=g-explicit-bool-comparison
- if middle_whitespace != ' ' and middle_whitespace != '':
- error(filename, linenum, 'whitespace/todo', 2,
- 'TODO(my_username) should be followed by a space')
-
- # If the comment contains an alphanumeric character, there
- # should be a space somewhere between it and the // unless
- # it's a /// or //! Doxygen comment.
- if (Match(r'//[^ ]*\w', comment) and
- not Match(r'(///|//\!)(\s+|$)', comment)):
- error(filename, linenum, 'whitespace/comments', 4,
- 'Should have a space between // and comment')
-
-
-def CheckAccess(filename, clean_lines, linenum, nesting_state, error):
- """Checks for improper use of DISALLOW* macros.
-
- Args:
- filename: The name of the current file.
- clean_lines: A CleansedLines instance containing the file.
- linenum: The number of the line to check.
- nesting_state: A NestingState instance which maintains information about
- the current stack of nested blocks being parsed.
- error: The function to call with any errors found.
- """
- line = clean_lines.elided[linenum] # get rid of comments and strings
-
- matched = Match((r'\s*(DISALLOW_COPY_AND_ASSIGN|'
- r'DISALLOW_IMPLICIT_CONSTRUCTORS)'), line)
- if not matched:
- return
- if nesting_state.stack and isinstance(nesting_state.stack[-1], _ClassInfo):
- if nesting_state.stack[-1].access != 'private':
- error(filename, linenum, 'readability/constructors', 3,
- '%s must be in the private: section' % matched.group(1))
-
- else:
- # Found DISALLOW* macro outside a class declaration, or perhaps it
- # was used inside a function when it should have been part of the
- # class declaration. We could issue a warning here, but it
- # probably resulted in a compiler error already.
- pass
-
-
-def CheckSpacing(filename, clean_lines, linenum, nesting_state, error):
- """Checks for the correctness of various spacing issues in the code.
-
- Things we check for: spaces around operators, spaces after
- if/for/while/switch, no spaces around parens in function calls, two
- spaces between code and comment, don't start a block with a blank
- line, don't end a function with a blank line, don't add a blank line
- after public/protected/private, don't have too many blank lines in a row.
-
- Args:
- filename: The name of the current file.
- clean_lines: A CleansedLines instance containing the file.
- linenum: The number of the line to check.
- nesting_state: A NestingState instance which maintains information about
- the current stack of nested blocks being parsed.
- error: The function to call with any errors found.
- """
-
- # Don't use "elided" lines here, otherwise we can't check commented lines.
- # Don't want to use "raw" either, because we don't want to check inside C++11
- # raw strings,
- raw = clean_lines.lines_without_raw_strings
- line = raw[linenum]
-
- # Before nixing comments, check if the line is blank for no good
- # reason. This includes the first line after a block is opened, and
- # blank lines at the end of a function (ie, right before a line like '}'
- #
- # Skip all the blank line checks if we are immediately inside a
- # namespace body. In other words, don't issue blank line warnings
- # for this block:
- # namespace {
- #
- # }
- #
- # A warning about missing end of namespace comments will be issued instead.
- #
- # Also skip blank line checks for 'extern "C"' blocks, which are formatted
- # like namespaces.
- if (IsBlankLine(line) and
- not nesting_state.InNamespaceBody() and
- not nesting_state.InExternC()):
- elided = clean_lines.elided
- prev_line = elided[linenum - 1]
- prevbrace = prev_line.rfind('{')
- # TODO(unknown): Don't complain if line before blank line, and line after,
- # both start with alnums and are indented the same amount.
- # This ignores whitespace at the start of a namespace block
- # because those are not usually indented.
- if prevbrace != -1 and prev_line[prevbrace:].find('}') == -1:
- # OK, we have a blank line at the start of a code block. Before we
- # complain, we check if it is an exception to the rule: The previous
- # non-empty line has the parameters of a function header that are indented
- # 4 spaces (because they did not fit in a 80 column line when placed on
- # the same line as the function name). We also check for the case where
- # the previous line is indented 6 spaces, which may happen when the
- # initializers of a constructor do not fit into a 80 column line.
- exception = False
- if Match(r' {6}\w', prev_line): # Initializer list?
- # We are looking for the opening column of initializer list, which
- # should be indented 4 spaces to cause 6 space indentation afterwards.
- search_position = linenum-2
- while (search_position >= 0
- and Match(r' {6}\w', elided[search_position])):
- search_position -= 1
- exception = (search_position >= 0
- and elided[search_position][:5] == ' :')
- else:
- # Search for the function arguments or an initializer list. We use a
- # simple heuristic here: If the line is indented 4 spaces; and we have a
- # closing paren, without the opening paren, followed by an opening brace
- # or colon (for initializer lists) we assume that it is the last line of
- # a function header. If we have a colon indented 4 spaces, it is an
- # initializer list.
- exception = (Match(r' {4}\w[^\(]*\)\s*(const\s*)?(\{\s*$|:)',
- prev_line)
- or Match(r' {4}:', prev_line))
-
- if not exception:
- error(filename, linenum, 'whitespace/blank_line', 2,
- 'Redundant blank line at the start of a code block '
- 'should be deleted.')
- # Ignore blank lines at the end of a block in a long if-else
- # chain, like this:
- # if (condition1) {
- # // Something followed by a blank line
- #
- # } else if (condition2) {
- # // Something else
- # }
- if linenum + 1 < clean_lines.NumLines():
- next_line = raw[linenum + 1]
- if (next_line
- and Match(r'\s*}', next_line)
- and next_line.find('} else ') == -1):
- error(filename, linenum, 'whitespace/blank_line', 3,
- 'Redundant blank line at the end of a code block '
- 'should be deleted.')
-
- matched = Match(r'\s*(public|protected|private):', prev_line)
- if matched:
- error(filename, linenum, 'whitespace/blank_line', 3,
- 'Do not leave a blank line after "%s:"' % matched.group(1))
-
- # Next, check comments
- next_line_start = 0
- if linenum + 1 < clean_lines.NumLines():
- next_line = raw[linenum + 1]
- next_line_start = len(next_line) - len(next_line.lstrip())
- CheckComment(line, filename, linenum, next_line_start, error)
-
- # get rid of comments and strings
- line = clean_lines.elided[linenum]
-
- # You shouldn't have spaces before your brackets, except maybe after
- # 'delete []' or 'return []() {};'
- if Search(r'\w\s+\[', line) and not Search(r'(?:delete|return)\s+\[', line):
- error(filename, linenum, 'whitespace/braces', 5,
- 'Extra space before [')
-
- # In range-based for, we wanted spaces before and after the colon, but
- # not around "::" tokens that might appear.
- if (Search(r'for *\(.*[^:]:[^: ]', line) or
- Search(r'for *\(.*[^: ]:[^:]', line)):
- error(filename, linenum, 'whitespace/forcolon', 2,
- 'Missing space around colon in range-based for loop')
-
-
-def CheckOperatorSpacing(filename, clean_lines, linenum, error):
- """Checks for horizontal spacing around operators.
-
- Args:
- filename: The name of the current file.
- clean_lines: A CleansedLines instance containing the file.
- linenum: The number of the line to check.
- error: The function to call with any errors found.
- """
- line = clean_lines.elided[linenum]
-
- # Don't try to do spacing checks for operator methods. Do this by
- # replacing the troublesome characters with something else,
- # preserving column position for all other characters.
- #
- # The replacement is done repeatedly to avoid false positives from
- # operators that call operators.
- while True:
- match = Match(r'^(.*\boperator\b)(\S+)(\s*\(.*)$', line)
- if match:
- line = match.group(1) + ('_' * len(match.group(2))) + match.group(3)
- else:
- break
-
- # We allow no-spaces around = within an if: "if ( (a=Foo()) == 0 )".
- # Otherwise not. Note we only check for non-spaces on *both* sides;
- # sometimes people put non-spaces on one side when aligning ='s among
- # many lines (not that this is behavior that I approve of...)
- if ((Search(r'[\w.]=', line) or
- Search(r'=[\w.]', line))
- and not Search(r'\b(if|while|for) ', line)
- # Operators taken from [lex.operators] in C++11 standard.
- and not Search(r'(>=|<=|==|!=|&=|\^=|\|=|\+=|\*=|\/=|\%=)', line)
- and not Search(r'operator=', line)):
- error(filename, linenum, 'whitespace/operators', 4,
- 'Missing spaces around =')
-
- # It's ok not to have spaces around binary operators like + - * /, but if
- # there's too little whitespace, we get concerned. It's hard to tell,
- # though, so we punt on this one for now. TODO.
-
- # You should always have whitespace around binary operators.
- #
- # Check <= and >= first to avoid false positives with < and >, then
- # check non-include lines for spacing around < and >.
- #
- # If the operator is followed by a comma, assume it's be used in a
- # macro context and don't do any checks. This avoids false
- # positives.
- #
- # Note that && is not included here. Those are checked separately
- # in CheckRValueReference
- match = Search(r'[^<>=!\s](==|!=|<=|>=|\|\|)[^<>=!\s,;\)]', line)
- if match:
- error(filename, linenum, 'whitespace/operators', 3,
- 'Missing spaces around %s' % match.group(1))
- elif not Match(r'#.*include', line):
- # Look for < that is not surrounded by spaces. This is only
- # triggered if both sides are missing spaces, even though
- # technically should should flag if at least one side is missing a
- # space. This is done to avoid some false positives with shifts.
- match = Match(r'^(.*[^\s<])<[^\s=<,]', line)
- if match:
- (_, _, end_pos) = CloseExpression(
- clean_lines, linenum, len(match.group(1)))
- if end_pos <= -1:
- error(filename, linenum, 'whitespace/operators', 3,
- 'Missing spaces around <')
-
- # Look for > that is not surrounded by spaces. Similar to the
- # above, we only trigger if both sides are missing spaces to avoid
- # false positives with shifts.
- match = Match(r'^(.*[^-\s>])>[^\s=>,]', line)
- if match:
- (_, _, start_pos) = ReverseCloseExpression(
- clean_lines, linenum, len(match.group(1)))
- if start_pos <= -1:
- error(filename, linenum, 'whitespace/operators', 3,
- 'Missing spaces around >')
-
- # We allow no-spaces around << when used like this: 10<<20, but
- # not otherwise (particularly, not when used as streams)
- #
- # We also allow operators following an opening parenthesis, since
- # those tend to be macros that deal with operators.
- match = Search(r'(operator|[^\s(<])(?:L|UL|ULL|l|ul|ull)?<<([^\s,=<])', line)
- if (match and not (match.group(1).isdigit() and match.group(2).isdigit()) and
- not (match.group(1) == 'operator' and match.group(2) == ';')):
- error(filename, linenum, 'whitespace/operators', 3,
- 'Missing spaces around <<')
-
- # We allow no-spaces around >> for almost anything. This is because
- # C++11 allows ">>" to close nested templates, which accounts for
- # most cases when ">>" is not followed by a space.
- #
- # We still warn on ">>" followed by alpha character, because that is
- # likely due to ">>" being used for right shifts, e.g.:
- # value >> alpha
- #
- # When ">>" is used to close templates, the alphanumeric letter that
- # follows would be part of an identifier, and there should still be
- # a space separating the template type and the identifier.
- # type<type<type>> alpha
- match = Search(r'>>[a-zA-Z_]', line)
- if match:
- error(filename, linenum, 'whitespace/operators', 3,
- 'Missing spaces around >>')
-
- # There shouldn't be space around unary operators
- match = Search(r'(!\s|~\s|[\s]--[\s;]|[\s]\+\+[\s;])', line)
- if match:
- error(filename, linenum, 'whitespace/operators', 4,
- 'Extra space for operator %s' % match.group(1))
-
-
-def CheckParenthesisSpacing(filename, clean_lines, linenum, error):
- """Checks for horizontal spacing around parentheses.
-
- Args:
- filename: The name of the current file.
- clean_lines: A CleansedLines instance containing the file.
- linenum: The number of the line to check.
- error: The function to call with any errors found.
- """
- line = clean_lines.elided[linenum]
-
- # No spaces after an if, while, switch, or for
- match = Search(r' (if\(|for\(|while\(|switch\()', line)
- if match:
- error(filename, linenum, 'whitespace/parens', 5,
- 'Missing space before ( in %s' % match.group(1))
-
- # For if/for/while/switch, the left and right parens should be
- # consistent about how many spaces are inside the parens, and
- # there should either be zero or one spaces inside the parens.
- # We don't want: "if ( foo)" or "if ( foo )".
- # Exception: "for ( ; foo; bar)" and "for (foo; bar; )" are allowed.
- match = Search(r'\b(if|for|while|switch)\s*'
- r'\(([ ]*)(.).*[^ ]+([ ]*)\)\s*{\s*$',
- line)
- if match:
- if len(match.group(2)) != len(match.group(4)):
- if not (match.group(3) == ';' and
- len(match.group(2)) == 1 + len(match.group(4)) or
- not match.group(2) and Search(r'\bfor\s*\(.*; \)', line)):
- error(filename, linenum, 'whitespace/parens', 5,
- 'Mismatching spaces inside () in %s' % match.group(1))
- if len(match.group(2)) not in [0, 1]:
- error(filename, linenum, 'whitespace/parens', 5,
- 'Should have zero or one spaces inside ( and ) in %s' %
- match.group(1))
-
-
-def CheckCommaSpacing(filename, clean_lines, linenum, error):
- """Checks for horizontal spacing near commas and semicolons.
-
- Args:
- filename: The name of the current file.
- clean_lines: A CleansedLines instance containing the file.
- linenum: The number of the line to check.
- error: The function to call with any errors found.
- """
- raw = clean_lines.lines_without_raw_strings
- line = clean_lines.elided[linenum]
-
- # You should always have a space after a comma (either as fn arg or operator)
- #
- # This does not apply when the non-space character following the
- # comma is another comma, since the only time when that happens is
- # for empty macro arguments.
- #
- # We run this check in two passes: first pass on elided lines to
- # verify that lines contain missing whitespaces, second pass on raw
- # lines to confirm that those missing whitespaces are not due to
- # elided comments.
- if (Search(r',[^,\s]', ReplaceAll(r'\boperator\s*,\s*\(', 'F(', line)) and
- Search(r',[^,\s]', raw[linenum])):
- error(filename, linenum, 'whitespace/comma', 3,
- 'Missing space after ,')
-
- # You should always have a space after a semicolon
- # except for few corner cases
- # TODO(unknown): clarify if 'if (1) { return 1;}' is requires one more
- # space after ;
- if Search(r';[^\s};\\)/]', line):
- error(filename, linenum, 'whitespace/semicolon', 3,
- 'Missing space after ;')
-
-
-def CheckBracesSpacing(filename, clean_lines, linenum, error):
- """Checks for horizontal spacing near commas.
-
- Args:
- filename: The name of the current file.
- clean_lines: A CleansedLines instance containing the file.
- linenum: The number of the line to check.
- error: The function to call with any errors found.
- """
- line = clean_lines.elided[linenum]
-
- # Except after an opening paren, or after another opening brace (in case of
- # an initializer list, for instance), you should have spaces before your
- # braces. And since you should never have braces at the beginning of a line,
- # this is an easy test.
- match = Match(r'^(.*[^ ({>]){', line)
- if match:
- # Try a bit harder to check for brace initialization. This
- # happens in one of the following forms:
- # Constructor() : initializer_list_{} { ... }
- # Constructor{}.MemberFunction()
- # Type variable{};
- # FunctionCall(type{}, ...);
- # LastArgument(..., type{});
- # LOG(INFO) << type{} << " ...";
- # map_of_type[{...}] = ...;
- # ternary = expr ? new type{} : nullptr;
- # OuterTemplate<InnerTemplateConstructor<Type>{}>
- #
- # We check for the character following the closing brace, and
- # silence the warning if it's one of those listed above, i.e.
- # "{.;,)<>]:".
- #
- # To account for nested initializer list, we allow any number of
- # closing braces up to "{;,)<". We can't simply silence the
- # warning on first sight of closing brace, because that would
- # cause false negatives for things that are not initializer lists.
- # Silence this: But not this:
- # Outer{ if (...) {
- # Inner{...} if (...){ // Missing space before {
- # }; }
- #
- # There is a false negative with this approach if people inserted
- # spurious semicolons, e.g. "if (cond){};", but we will catch the
- # spurious semicolon with a separate check.
- (endline, endlinenum, endpos) = CloseExpression(
- clean_lines, linenum, len(match.group(1)))
- trailing_text = ''
- if endpos > -1:
- trailing_text = endline[endpos:]
- for offset in xrange(endlinenum + 1,
- min(endlinenum + 3, clean_lines.NumLines() - 1)):
- trailing_text += clean_lines.elided[offset]
- if not Match(r'^[\s}]*[{.;,)<>\]:]', trailing_text):
- error(filename, linenum, 'whitespace/braces', 5,
- 'Missing space before {')
-
- # Make sure '} else {' has spaces.
- if Search(r'}else', line):
- error(filename, linenum, 'whitespace/braces', 5,
- 'Missing space before else')
-
- # You shouldn't have a space before a semicolon at the end of the line.
- # There's a special case for "for" since the style guide allows space before
- # the semicolon there.
- if Search(r':\s*;\s*$', line):
- error(filename, linenum, 'whitespace/semicolon', 5,
- 'Semicolon defining empty statement. Use {} instead.')
- elif Search(r'^\s*;\s*$', line):
- error(filename, linenum, 'whitespace/semicolon', 5,
- 'Line contains only semicolon. If this should be an empty statement, '
- 'use {} instead.')
- elif (Search(r'\s+;\s*$', line) and
- not Search(r'\bfor\b', line)):
- error(filename, linenum, 'whitespace/semicolon', 5,
- 'Extra space before last semicolon. If this should be an empty '
- 'statement, use {} instead.')
-
-
-def IsDecltype(clean_lines, linenum, column):
- """Check if the token ending on (linenum, column) is decltype().
-
- Args:
- clean_lines: A CleansedLines instance containing the file.
- linenum: the number of the line to check.
- column: end column of the token to check.
- Returns:
- True if this token is decltype() expression, False otherwise.
- """
- (text, _, start_col) = ReverseCloseExpression(clean_lines, linenum, column)
- if start_col < 0:
- return False
- if Search(r'\bdecltype\s*$', text[0:start_col]):
- return True
- return False
-
-
-def IsTemplateParameterList(clean_lines, linenum, column):
- """Check if the token ending on (linenum, column) is the end of template<>.
-
- Args:
- clean_lines: A CleansedLines instance containing the file.
- linenum: the number of the line to check.
- column: end column of the token to check.
- Returns:
- True if this token is end of a template parameter list, False otherwise.
- """
- (_, startline, startpos) = ReverseCloseExpression(
- clean_lines, linenum, column)
- if (startpos > -1 and
- Search(r'\btemplate\s*$', clean_lines.elided[startline][0:startpos])):
- return True
- return False
-
-
-def IsRValueType(typenames, clean_lines, nesting_state, linenum, column):
- """Check if the token ending on (linenum, column) is a type.
-
- Assumes that text to the right of the column is "&&" or a function
- name.
-
- Args:
- typenames: set of type names from template-argument-list.
- clean_lines: A CleansedLines instance containing the file.
- nesting_state: A NestingState instance which maintains information about
- the current stack of nested blocks being parsed.
- linenum: the number of the line to check.
- column: end column of the token to check.
- Returns:
- True if this token is a type, False if we are not sure.
- """
- prefix = clean_lines.elided[linenum][0:column]
-
- # Get one word to the left. If we failed to do so, this is most
- # likely not a type, since it's unlikely that the type name and "&&"
- # would be split across multiple lines.
- match = Match(r'^(.*)(\b\w+|[>*)&])\s*$', prefix)
- if not match:
- return False
-
- # Check text following the token. If it's "&&>" or "&&," or "&&...", it's
- # most likely a rvalue reference used inside a template.
- suffix = clean_lines.elided[linenum][column:]
- if Match(r'&&\s*(?:[>,]|\.\.\.)', suffix):
- return True
-
- # Check for known types and end of templates:
- # int&& variable
- # vector<int>&& variable
- #
- # Because this function is called recursively, we also need to
- # recognize pointer and reference types:
- # int* Function()
- # int& Function()
- if (match.group(2) in typenames or
- match.group(2) in ['char', 'char16_t', 'char32_t', 'wchar_t', 'bool',
- 'short', 'int', 'long', 'signed', 'unsigned',
- 'float', 'double', 'void', 'auto', '>', '*', '&']):
- return True
-
- # If we see a close parenthesis, look for decltype on the other side.
- # decltype would unambiguously identify a type, anything else is
- # probably a parenthesized expression and not a type.
- if match.group(2) == ')':
- return IsDecltype(
- clean_lines, linenum, len(match.group(1)) + len(match.group(2)) - 1)
-
- # Check for casts and cv-qualifiers.
- # match.group(1) remainder
- # -------------- ---------
- # const_cast< type&&
- # const type&&
- # type const&&
- if Search(r'\b(?:const_cast\s*<|static_cast\s*<|dynamic_cast\s*<|'
- r'reinterpret_cast\s*<|\w+\s)\s*$',
- match.group(1)):
- return True
-
- # Look for a preceding symbol that might help differentiate the context.
- # These are the cases that would be ambiguous:
- # match.group(1) remainder
- # -------------- ---------
- # Call ( expression &&
- # Declaration ( type&&
- # sizeof ( type&&
- # if ( expression &&
- # while ( expression &&
- # for ( type&&
- # for( ; expression &&
- # statement ; type&&
- # block { type&&
- # constructor { expression &&
- start = linenum
- line = match.group(1)
- match_symbol = None
- while start >= 0:
- # We want to skip over identifiers and commas to get to a symbol.
- # Commas are skipped so that we can find the opening parenthesis
- # for function parameter lists.
- match_symbol = Match(r'^(.*)([^\w\s,])[\w\s,]*$', line)
- if match_symbol:
- break
- start -= 1
- line = clean_lines.elided[start]
-
- if not match_symbol:
- # Probably the first statement in the file is an rvalue reference
- return True
-
- if match_symbol.group(2) == '}':
- # Found closing brace, probably an indicate of this:
- # block{} type&&
- return True
-
- if match_symbol.group(2) == ';':
- # Found semicolon, probably one of these:
- # for(; expression &&
- # statement; type&&
-
- # Look for the previous 'for(' in the previous lines.
- before_text = match_symbol.group(1)
- for i in xrange(start - 1, max(start - 6, 0), -1):
- before_text = clean_lines.elided[i] + before_text
- if Search(r'for\s*\([^{};]*$', before_text):
- # This is the condition inside a for-loop
- return False
-
- # Did not find a for-init-statement before this semicolon, so this
- # is probably a new statement and not a condition.
- return True
-
- if match_symbol.group(2) == '{':
- # Found opening brace, probably one of these:
- # block{ type&& = ... ; }
- # constructor{ expression && expression }
-
- # Look for a closing brace or a semicolon. If we see a semicolon
- # first, this is probably a rvalue reference.
- line = clean_lines.elided[start][0:len(match_symbol.group(1)) + 1]
- end = start
- depth = 1
- while True:
- for ch in line:
- if ch == ';':
- return True
- elif ch == '{':
- depth += 1
- elif ch == '}':
- depth -= 1
- if depth == 0:
- return False
- end += 1
- if end >= clean_lines.NumLines():
- break
- line = clean_lines.elided[end]
- # Incomplete program?
- return False
-
- if match_symbol.group(2) == '(':
- # Opening parenthesis. Need to check what's to the left of the
- # parenthesis. Look back one extra line for additional context.
- before_text = match_symbol.group(1)
- if linenum > 1:
- before_text = clean_lines.elided[linenum - 1] + before_text
- before_text = match_symbol.group(1)
-
- # Patterns that are likely to be types:
- # [](type&&
- # for (type&&
- # sizeof(type&&
- # operator=(type&&
- #
- if Search(r'(?:\]|\bfor|\bsizeof|\boperator\s*\S+\s*)\s*$', before_text):
- return True
-
- # Patterns that are likely to be expressions:
- # if (expression &&
- # while (expression &&
- # : initializer(expression &&
- # , initializer(expression &&
- # ( FunctionCall(expression &&
- # + FunctionCall(expression &&
- # + (expression &&
- #
- # The last '+' represents operators such as '+' and '-'.
- if Search(r'(?:\bif|\bwhile|[-+=%^(<!?:,&*]\s*)$', before_text):
- return False
-
- # Something else. Check that tokens to the left look like
- # return_type function_name
- match_func = Match(r'^(.*\S.*)\s+\w(?:\w|::)*(?:<[^<>]*>)?\s*$',
- match_symbol.group(1))
- if match_func:
- # Check for constructors, which don't have return types.
- if Search(r'\b(?:explicit|inline)$', match_func.group(1)):
- return True
- implicit_constructor = Match(r'\s*(\w+)\((?:const\s+)?(\w+)', prefix)
- if (implicit_constructor and
- implicit_constructor.group(1) == implicit_constructor.group(2)):
- return True
- return IsRValueType(typenames, clean_lines, nesting_state, linenum,
- len(match_func.group(1)))
-
- # Nothing before the function name. If this is inside a block scope,
- # this is probably a function call.
- return not (nesting_state.previous_stack_top and
- nesting_state.previous_stack_top.IsBlockInfo())
-
- if match_symbol.group(2) == '>':
- # Possibly a closing bracket, check that what's on the other side
- # looks like the start of a template.
- return IsTemplateParameterList(
- clean_lines, start, len(match_symbol.group(1)))
-
- # Some other symbol, usually something like "a=b&&c". This is most
- # likely not a type.
- return False
-
-
-def IsDeletedOrDefault(clean_lines, linenum):
- """Check if current constructor or operator is deleted or default.
-
- Args:
- clean_lines: A CleansedLines instance containing the file.
- linenum: The number of the line to check.
- Returns:
- True if this is a deleted or default constructor.
- """
- open_paren = clean_lines.elided[linenum].find('(')
- if open_paren < 0:
- return False
- (close_line, _, close_paren) = CloseExpression(
- clean_lines, linenum, open_paren)
- if close_paren < 0:
- return False
- return Match(r'\s*=\s*(?:delete|default)\b', close_line[close_paren:])
-
-
-def IsRValueAllowed(clean_lines, linenum, typenames):
- """Check if RValue reference is allowed on a particular line.
-
- Args:
- clean_lines: A CleansedLines instance containing the file.
- linenum: The number of the line to check.
- typenames: set of type names from template-argument-list.
- Returns:
- True if line is within the region where RValue references are allowed.
- """
- # Allow region marked by PUSH/POP macros
- for i in xrange(linenum, 0, -1):
- line = clean_lines.elided[i]
- if Match(r'GOOGLE_ALLOW_RVALUE_REFERENCES_(?:PUSH|POP)', line):
- if not line.endswith('PUSH'):
- return False
- for j in xrange(linenum, clean_lines.NumLines(), 1):
- line = clean_lines.elided[j]
- if Match(r'GOOGLE_ALLOW_RVALUE_REFERENCES_(?:PUSH|POP)', line):
- return line.endswith('POP')
-
- # Allow operator=
- line = clean_lines.elided[linenum]
- if Search(r'\boperator\s*=\s*\(', line):
- return IsDeletedOrDefault(clean_lines, linenum)
-
- # Allow constructors
- match = Match(r'\s*(?:[\w<>]+::)*([\w<>]+)\s*::\s*([\w<>]+)\s*\(', line)
- if match and match.group(1) == match.group(2):
- return IsDeletedOrDefault(clean_lines, linenum)
- if Search(r'\b(?:explicit|inline)\s+[\w<>]+\s*\(', line):
- return IsDeletedOrDefault(clean_lines, linenum)
-
- if Match(r'\s*[\w<>]+\s*\(', line):
- previous_line = 'ReturnType'
- if linenum > 0:
- previous_line = clean_lines.elided[linenum - 1]
- if Match(r'^\s*$', previous_line) or Search(r'[{}:;]\s*$', previous_line):
- return IsDeletedOrDefault(clean_lines, linenum)
-
- # Reject types not mentioned in template-argument-list
- while line:
- match = Match(r'^.*?(\w+)\s*&&(.*)$', line)
- if not match:
- break
- if match.group(1) not in typenames:
- return False
- line = match.group(2)
-
- # All RValue types that were in template-argument-list should have
- # been removed by now. Those were allowed, assuming that they will
- # be forwarded.
- #
- # If there are no remaining RValue types left (i.e. types that were
- # not found in template-argument-list), flag those as not allowed.
- return line.find('&&') < 0
-
-
-def GetTemplateArgs(clean_lines, linenum):
- """Find list of template arguments associated with this function declaration.
-
- Args:
- clean_lines: A CleansedLines instance containing the file.
- linenum: Line number containing the start of the function declaration,
- usually one line after the end of the template-argument-list.
- Returns:
- Set of type names, or empty set if this does not appear to have
- any template parameters.
- """
- # Find start of function
- func_line = linenum
- while func_line > 0:
- line = clean_lines.elided[func_line]
- if Match(r'^\s*$', line):
- return set()
- if line.find('(') >= 0:
- break
- func_line -= 1
- if func_line == 0:
- return set()
-
- # Collapse template-argument-list into a single string
- argument_list = ''
- match = Match(r'^(\s*template\s*)<', clean_lines.elided[func_line])
- if match:
- # template-argument-list on the same line as function name
- start_col = len(match.group(1))
- _, end_line, end_col = CloseExpression(clean_lines, func_line, start_col)
- if end_col > -1 and end_line == func_line:
- start_col += 1 # Skip the opening bracket
- argument_list = clean_lines.elided[func_line][start_col:end_col]
-
- elif func_line > 1:
- # template-argument-list one line before function name
- match = Match(r'^(.*)>\s*$', clean_lines.elided[func_line - 1])
- if match:
- end_col = len(match.group(1))
- _, start_line, start_col = ReverseCloseExpression(
- clean_lines, func_line - 1, end_col)
- if start_col > -1:
- start_col += 1 # Skip the opening bracket
- while start_line < func_line - 1:
- argument_list += clean_lines.elided[start_line][start_col:]
- start_col = 0
- start_line += 1
- argument_list += clean_lines.elided[func_line - 1][start_col:end_col]
-
- if not argument_list:
- return set()
-
- # Extract type names
- typenames = set()
- while True:
- match = Match(r'^[,\s]*(?:typename|class)(?:\.\.\.)?\s+(\w+)(.*)$',
- argument_list)
- if not match:
- break
- typenames.add(match.group(1))
- argument_list = match.group(2)
- return typenames
-
-
-def CheckRValueReference(filename, clean_lines, linenum, nesting_state, error):
- """Check for rvalue references.
-
- Args:
- filename: The name of the current file.
- clean_lines: A CleansedLines instance containing the file.
- linenum: The number of the line to check.
- nesting_state: A NestingState instance which maintains information about
- the current stack of nested blocks being parsed.
- error: The function to call with any errors found.
- """
- # Find lines missing spaces around &&.
- # TODO(unknown): currently we don't check for rvalue references
- # with spaces surrounding the && to avoid false positives with
- # boolean expressions.
- line = clean_lines.elided[linenum]
- match = Match(r'^(.*\S)&&', line)
- if not match:
- match = Match(r'(.*)&&\S', line)
- if (not match) or '(&&)' in line or Search(r'\boperator\s*$', match.group(1)):
- return
-
- # Either poorly formed && or an rvalue reference, check the context
- # to get a more accurate error message. Mostly we want to determine
- # if what's to the left of "&&" is a type or not.
- typenames = GetTemplateArgs(clean_lines, linenum)
- and_pos = len(match.group(1))
- if IsRValueType(typenames, clean_lines, nesting_state, linenum, and_pos):
- if not IsRValueAllowed(clean_lines, linenum, typenames):
- error(filename, linenum, 'build/c++11', 3,
- 'RValue references are an unapproved C++ feature.')
- else:
- error(filename, linenum, 'whitespace/operators', 3,
- 'Missing spaces around &&')
-
-
-def CheckSectionSpacing(filename, clean_lines, class_info, linenum, error):
- """Checks for additional blank line issues related to sections.
-
- Currently the only thing checked here is blank line before protected/private.
-
- Args:
- filename: The name of the current file.
- clean_lines: A CleansedLines instance containing the file.
- class_info: A _ClassInfo objects.
- linenum: The number of the line to check.
- error: The function to call with any errors found.
- """
- # Skip checks if the class is small, where small means 25 lines or less.
- # 25 lines seems like a good cutoff since that's the usual height of
- # terminals, and any class that can't fit in one screen can't really
- # be considered "small".
- #
- # Also skip checks if we are on the first line. This accounts for
- # classes that look like
- # class Foo { public: ... };
- #
- # If we didn't find the end of the class, last_line would be zero,
- # and the check will be skipped by the first condition.
- if (class_info.last_line - class_info.starting_linenum <= 24 or
- linenum <= class_info.starting_linenum):
- return
-
- matched = Match(r'\s*(public|protected|private):', clean_lines.lines[linenum])
- if matched:
- # Issue warning if the line before public/protected/private was
- # not a blank line, but don't do this if the previous line contains
- # "class" or "struct". This can happen two ways:
- # - We are at the beginning of the class.
- # - We are forward-declaring an inner class that is semantically
- # private, but needed to be public for implementation reasons.
- # Also ignores cases where the previous line ends with a backslash as can be
- # common when defining classes in C macros.
- prev_line = clean_lines.lines[linenum - 1]
- if (not IsBlankLine(prev_line) and
- not Search(r'\b(class|struct)\b', prev_line) and
- not Search(r'\\$', prev_line)):
- # Try a bit harder to find the beginning of the class. This is to
- # account for multi-line base-specifier lists, e.g.:
- # class Derived
- # : public Base {
- end_class_head = class_info.starting_linenum
- for i in range(class_info.starting_linenum, linenum):
- if Search(r'\{\s*$', clean_lines.lines[i]):
- end_class_head = i
- break
- if end_class_head < linenum - 1:
- error(filename, linenum, 'whitespace/blank_line', 3,
- '"%s:" should be preceded by a blank line' % matched.group(1))
-
-
-def GetPreviousNonBlankLine(clean_lines, linenum):
- """Return the most recent non-blank line and its line number.
-
- Args:
- clean_lines: A CleansedLines instance containing the file contents.
- linenum: The number of the line to check.
-
- Returns:
- A tuple with two elements. The first element is the contents of the last
- non-blank line before the current line, or the empty string if this is the
- first non-blank line. The second is the line number of that line, or -1
- if this is the first non-blank line.
- """
-
- prevlinenum = linenum - 1
- while prevlinenum >= 0:
- prevline = clean_lines.elided[prevlinenum]
- if not IsBlankLine(prevline): # if not a blank line...
- return (prevline, prevlinenum)
- prevlinenum -= 1
- return ('', -1)
-
-
-def CheckBraces(filename, clean_lines, linenum, error):
- """Looks for misplaced braces (e.g. at the end of line).
-
- Args:
- filename: The name of the current file.
- clean_lines: A CleansedLines instance containing the file.
- linenum: The number of the line to check.
- error: The function to call with any errors found.
- """
-
- line = clean_lines.elided[linenum] # get rid of comments and strings
-
- if Match(r'\s*{\s*$', line):
- # We allow an open brace to start a line in the case where someone is using
- # braces in a block to explicitly create a new scope, which is commonly used
- # to control the lifetime of stack-allocated variables. Braces are also
- # used for brace initializers inside function calls. We don't detect this
- # perfectly: we just don't complain if the last non-whitespace character on
- # the previous non-blank line is ',', ';', ':', '(', '{', or '}', or if the
- # previous line starts a preprocessor block.
- prevline = GetPreviousNonBlankLine(clean_lines, linenum)[0]
- if (not Search(r'[,;:}{(]\s*$', prevline) and
- not Match(r'\s*#', prevline)):
- error(filename, linenum, 'whitespace/braces', 4,
- '{ should almost always be at the end of the previous line')
-
- # An else clause should be on the same line as the preceding closing brace.
- if Match(r'\s*else\b\s*(?:if\b|\{|$)', line):
- prevline = GetPreviousNonBlankLine(clean_lines, linenum)[0]
- if Match(r'\s*}\s*$', prevline):
- error(filename, linenum, 'whitespace/newline', 4,
- 'An else should appear on the same line as the preceding }')
-
- # If braces come on one side of an else, they should be on both.
- # However, we have to worry about "else if" that spans multiple lines!
- if Search(r'else if\s*\(', line): # could be multi-line if
- brace_on_left = bool(Search(r'}\s*else if\s*\(', line))
- # find the ( after the if
- pos = line.find('else if')
- pos = line.find('(', pos)
- if pos > 0:
- (endline, _, endpos) = CloseExpression(clean_lines, linenum, pos)
- brace_on_right = endline[endpos:].find('{') != -1
- if brace_on_left != brace_on_right: # must be brace after if
- error(filename, linenum, 'readability/braces', 5,
- 'If an else has a brace on one side, it should have it on both')
- elif Search(r'}\s*else[^{]*$', line) or Match(r'[^}]*else\s*{', line):
- error(filename, linenum, 'readability/braces', 5,
- 'If an else has a brace on one side, it should have it on both')
-
- # Likewise, an else should never have the else clause on the same line
- if Search(r'\belse [^\s{]', line) and not Search(r'\belse if\b', line):
- error(filename, linenum, 'whitespace/newline', 4,
- 'Else clause should never be on same line as else (use 2 lines)')
-
- # In the same way, a do/while should never be on one line
- if Match(r'\s*do [^\s{]', line):
- error(filename, linenum, 'whitespace/newline', 4,
- 'do/while clauses should not be on a single line')
-
- # Check single-line if/else bodies. The style guide says 'curly braces are not
- # required for single-line statements'. We additionally allow multi-line,
- # single statements, but we reject anything with more than one semicolon in
- # it. This means that the first semicolon after the if should be at the end of
- # its line, and the line after that should have an indent level equal to or
- # lower than the if. We also check for ambiguous if/else nesting without
- # braces.
- if_else_match = Search(r'\b(if\s*\(|else\b)', line)
- if if_else_match and not Match(r'\s*#', line):
- if_indent = GetIndentLevel(line)
- endline, endlinenum, endpos = line, linenum, if_else_match.end()
- if_match = Search(r'\bif\s*\(', line)
- if if_match:
- # This could be a multiline if condition, so find the end first.
- pos = if_match.end() - 1
- (endline, endlinenum, endpos) = CloseExpression(clean_lines, linenum, pos)
- # Check for an opening brace, either directly after the if or on the next
- # line. If found, this isn't a single-statement conditional.
- if (not Match(r'\s*{', endline[endpos:])
- and not (Match(r'\s*$', endline[endpos:])
- and endlinenum < (len(clean_lines.elided) - 1)
- and Match(r'\s*{', clean_lines.elided[endlinenum + 1]))):
- while (endlinenum < len(clean_lines.elided)
- and ';' not in clean_lines.elided[endlinenum][endpos:]):
- endlinenum += 1
- endpos = 0
- if endlinenum < len(clean_lines.elided):
- endline = clean_lines.elided[endlinenum]
- # We allow a mix of whitespace and closing braces (e.g. for one-liner
- # methods) and a single \ after the semicolon (for macros)
- endpos = endline.find(';')
- if not Match(r';[\s}]*(\\?)$', endline[endpos:]):
- # Semicolon isn't the last character, there's something trailing.
- # Output a warning if the semicolon is not contained inside
- # a lambda expression.
- if not Match(r'^[^{};]*\[[^\[\]]*\][^{}]*\{[^{}]*\}\s*\)*[;,]\s*$',
- endline):
- error(filename, linenum, 'readability/braces', 4,
- 'If/else bodies with multiple statements require braces')
- elif endlinenum < len(clean_lines.elided) - 1:
- # Make sure the next line is dedented
- next_line = clean_lines.elided[endlinenum + 1]
- next_indent = GetIndentLevel(next_line)
- # With ambiguous nested if statements, this will error out on the
- # if that *doesn't* match the else, regardless of whether it's the
- # inner one or outer one.
- if (if_match and Match(r'\s*else\b', next_line)
- and next_indent != if_indent):
- error(filename, linenum, 'readability/braces', 4,
- 'Else clause should be indented at the same level as if. '
- 'Ambiguous nested if/else chains require braces.')
- elif next_indent > if_indent:
- error(filename, linenum, 'readability/braces', 4,
- 'If/else bodies with multiple statements require braces')
-
-
-def CheckTrailingSemicolon(filename, clean_lines, linenum, error):
- """Looks for redundant trailing semicolon.
-
- Args:
- filename: The name of the current file.
- clean_lines: A CleansedLines instance containing the file.
- linenum: The number of the line to check.
- error: The function to call with any errors found.
- """
-
- line = clean_lines.elided[linenum]
-
- # Block bodies should not be followed by a semicolon. Due to C++11
- # brace initialization, there are more places where semicolons are
- # required than not, so we use a whitelist approach to check these
- # rather than a blacklist. These are the places where "};" should
- # be replaced by just "}":
- # 1. Some flavor of block following closing parenthesis:
- # for (;;) {};
- # while (...) {};
- # switch (...) {};
- # Function(...) {};
- # if (...) {};
- # if (...) else if (...) {};
- #
- # 2. else block:
- # if (...) else {};
- #
- # 3. const member function:
- # Function(...) const {};
- #
- # 4. Block following some statement:
- # x = 42;
- # {};
- #
- # 5. Block at the beginning of a function:
- # Function(...) {
- # {};
- # }
- #
- # Note that naively checking for the preceding "{" will also match
- # braces inside multi-dimensional arrays, but this is fine since
- # that expression will not contain semicolons.
- #
- # 6. Block following another block:
- # while (true) {}
- # {};
- #
- # 7. End of namespaces:
- # namespace {};
- #
- # These semicolons seems far more common than other kinds of
- # redundant semicolons, possibly due to people converting classes
- # to namespaces. For now we do not warn for this case.
- #
- # Try matching case 1 first.
- match = Match(r'^(.*\)\s*)\{', line)
- if match:
- # Matched closing parenthesis (case 1). Check the token before the
- # matching opening parenthesis, and don't warn if it looks like a
- # macro. This avoids these false positives:
- # - macro that defines a base class
- # - multi-line macro that defines a base class
- # - macro that defines the whole class-head
- #
- # But we still issue warnings for macros that we know are safe to
- # warn, specifically:
- # - TEST, TEST_F, TEST_P, MATCHER, MATCHER_P
- # - TYPED_TEST
- # - INTERFACE_DEF
- # - EXCLUSIVE_LOCKS_REQUIRED, SHARED_LOCKS_REQUIRED, LOCKS_EXCLUDED:
- #
- # We implement a whitelist of safe macros instead of a blacklist of
- # unsafe macros, even though the latter appears less frequently in
- # google code and would have been easier to implement. This is because
- # the downside for getting the whitelist wrong means some extra
- # semicolons, while the downside for getting the blacklist wrong
- # would result in compile errors.
- #
- # In addition to macros, we also don't want to warn on
- # - Compound literals
- # - Lambdas
- # - alignas specifier with anonymous structs:
- closing_brace_pos = match.group(1).rfind(')')
- opening_parenthesis = ReverseCloseExpression(
- clean_lines, linenum, closing_brace_pos)
- if opening_parenthesis[2] > -1:
- line_prefix = opening_parenthesis[0][0:opening_parenthesis[2]]
- macro = Search(r'\b([A-Z_]+)\s*$', line_prefix)
- func = Match(r'^(.*\])\s*$', line_prefix)
- if ((macro and
- macro.group(1) not in (
- 'TEST', 'TEST_F', 'MATCHER', 'MATCHER_P', 'TYPED_TEST',
- 'EXCLUSIVE_LOCKS_REQUIRED', 'SHARED_LOCKS_REQUIRED',
- 'LOCKS_EXCLUDED', 'INTERFACE_DEF')) or
- (func and not Search(r'\boperator\s*\[\s*\]', func.group(1))) or
- Search(r'\b(?:struct|union)\s+alignas\s*$', line_prefix) or
- Search(r'\s+=\s*$', line_prefix)):
- match = None
- if (match and
- opening_parenthesis[1] > 1 and
- Search(r'\]\s*$', clean_lines.elided[opening_parenthesis[1] - 1])):
- # Multi-line lambda-expression
- match = None
-
- else:
- # Try matching cases 2-3.
- match = Match(r'^(.*(?:else|\)\s*const)\s*)\{', line)
- if not match:
- # Try matching cases 4-6. These are always matched on separate lines.
- #
- # Note that we can't simply concatenate the previous line to the
- # current line and do a single match, otherwise we may output
- # duplicate warnings for the blank line case:
- # if (cond) {
- # // blank line
- # }
- prevline = GetPreviousNonBlankLine(clean_lines, linenum)[0]
- if prevline and Search(r'[;{}]\s*$', prevline):
- match = Match(r'^(\s*)\{', line)
-
- # Check matching closing brace
- if match:
- (endline, endlinenum, endpos) = CloseExpression(
- clean_lines, linenum, len(match.group(1)))
- if endpos > -1 and Match(r'^\s*;', endline[endpos:]):
- # Current {} pair is eligible for semicolon check, and we have found
- # the redundant semicolon, output warning here.
- #
- # Note: because we are scanning forward for opening braces, and
- # outputting warnings for the matching closing brace, if there are
- # nested blocks with trailing semicolons, we will get the error
- # messages in reversed order.
- error(filename, endlinenum, 'readability/braces', 4,
- "You don't need a ; after a }")
-
-
-def CheckEmptyBlockBody(filename, clean_lines, linenum, error):
- """Look for empty loop/conditional body with only a single semicolon.
-
- Args:
- filename: The name of the current file.
- clean_lines: A CleansedLines instance containing the file.
- linenum: The number of the line to check.
- error: The function to call with any errors found.
- """
-
- # Search for loop keywords at the beginning of the line. Because only
- # whitespaces are allowed before the keywords, this will also ignore most
- # do-while-loops, since those lines should start with closing brace.
- #
- # We also check "if" blocks here, since an empty conditional block
- # is likely an error.
- line = clean_lines.elided[linenum]
- matched = Match(r'\s*(for|while|if)\s*\(', line)
- if matched:
- # Find the end of the conditional expression
- (end_line, end_linenum, end_pos) = CloseExpression(
- clean_lines, linenum, line.find('('))
-
- # Output warning if what follows the condition expression is a semicolon.
- # No warning for all other cases, including whitespace or newline, since we
- # have a separate check for semicolons preceded by whitespace.
- if end_pos >= 0 and Match(r';', end_line[end_pos:]):
- if matched.group(1) == 'if':
- error(filename, end_linenum, 'whitespace/empty_conditional_body', 5,
- 'Empty conditional bodies should use {}')
- else:
- error(filename, end_linenum, 'whitespace/empty_loop_body', 5,
- 'Empty loop bodies should use {} or continue')
-
-
-def FindCheckMacro(line):
- """Find a replaceable CHECK-like macro.
-
- Args:
- line: line to search on.
- Returns:
- (macro name, start position), or (None, -1) if no replaceable
- macro is found.
- """
- for macro in _CHECK_MACROS:
- i = line.find(macro)
- if i >= 0:
- # Find opening parenthesis. Do a regular expression match here
- # to make sure that we are matching the expected CHECK macro, as
- # opposed to some other macro that happens to contain the CHECK
- # substring.
- matched = Match(r'^(.*\b' + macro + r'\s*)\(', line)
- if not matched:
- continue
- return (macro, len(matched.group(1)))
- return (None, -1)
-
-
-def CheckCheck(filename, clean_lines, linenum, error):
- """Checks the use of CHECK and EXPECT macros.
-
- Args:
- filename: The name of the current file.
- clean_lines: A CleansedLines instance containing the file.
- linenum: The number of the line to check.
- error: The function to call with any errors found.
- """
-
- # Decide the set of replacement macros that should be suggested
- lines = clean_lines.elided
- (check_macro, start_pos) = FindCheckMacro(lines[linenum])
- if not check_macro:
- return
-
- # Find end of the boolean expression by matching parentheses
- (last_line, end_line, end_pos) = CloseExpression(
- clean_lines, linenum, start_pos)
- if end_pos < 0:
- return
-
- # If the check macro is followed by something other than a
- # semicolon, assume users will log their own custom error messages
- # and don't suggest any replacements.
- if not Match(r'\s*;', last_line[end_pos:]):
- return
-
- if linenum == end_line:
- expression = lines[linenum][start_pos + 1:end_pos - 1]
- else:
- expression = lines[linenum][start_pos + 1:]
- for i in xrange(linenum + 1, end_line):
- expression += lines[i]
- expression += last_line[0:end_pos - 1]
-
- # Parse expression so that we can take parentheses into account.
- # This avoids false positives for inputs like "CHECK((a < 4) == b)",
- # which is not replaceable by CHECK_LE.
- lhs = ''
- rhs = ''
- operator = None
- while expression:
- matched = Match(r'^\s*(<<|<<=|>>|>>=|->\*|->|&&|\|\||'
- r'==|!=|>=|>|<=|<|\()(.*)$', expression)
- if matched:
- token = matched.group(1)
- if token == '(':
- # Parenthesized operand
- expression = matched.group(2)
- (end, _) = FindEndOfExpressionInLine(expression, 0, ['('])
- if end < 0:
- return # Unmatched parenthesis
- lhs += '(' + expression[0:end]
- expression = expression[end:]
- elif token in ('&&', '||'):
- # Logical and/or operators. This means the expression
- # contains more than one term, for example:
- # CHECK(42 < a && a < b);
- #
- # These are not replaceable with CHECK_LE, so bail out early.
- return
- elif token in ('<<', '<<=', '>>', '>>=', '->*', '->'):
- # Non-relational operator
- lhs += token
- expression = matched.group(2)
- else:
- # Relational operator
- operator = token
- rhs = matched.group(2)
- break
- else:
- # Unparenthesized operand. Instead of appending to lhs one character
- # at a time, we do another regular expression match to consume several
- # characters at once if possible. Trivial benchmark shows that this
- # is more efficient when the operands are longer than a single
- # character, which is generally the case.
- matched = Match(r'^([^-=!<>()&|]+)(.*)$', expression)
- if not matched:
- matched = Match(r'^(\s*\S)(.*)$', expression)
- if not matched:
- break
- lhs += matched.group(1)
- expression = matched.group(2)
-
- # Only apply checks if we got all parts of the boolean expression
- if not (lhs and operator and rhs):
- return
-
- # Check that rhs do not contain logical operators. We already know
- # that lhs is fine since the loop above parses out && and ||.
- if rhs.find('&&') > -1 or rhs.find('||') > -1:
- return
-
- # At least one of the operands must be a constant literal. This is
- # to avoid suggesting replacements for unprintable things like
- # CHECK(variable != iterator)
- #
- # The following pattern matches decimal, hex integers, strings, and
- # characters (in that order).
- lhs = lhs.strip()
- rhs = rhs.strip()
- match_constant = r'^([-+]?(\d+|0[xX][0-9a-fA-F]+)[lLuU]{0,3}|".*"|\'.*\')$'
- if Match(match_constant, lhs) or Match(match_constant, rhs):
- # Note: since we know both lhs and rhs, we can provide a more
- # descriptive error message like:
- # Consider using CHECK_EQ(x, 42) instead of CHECK(x == 42)
- # Instead of:
- # Consider using CHECK_EQ instead of CHECK(a == b)
- #
- # We are still keeping the less descriptive message because if lhs
- # or rhs gets long, the error message might become unreadable.
- error(filename, linenum, 'readability/check', 2,
- 'Consider using %s instead of %s(a %s b)' % (
- _CHECK_REPLACEMENT[check_macro][operator],
- check_macro, operator))
-
-
-def CheckAltTokens(filename, clean_lines, linenum, error):
- """Check alternative keywords being used in boolean expressions.
-
- Args:
- filename: The name of the current file.
- clean_lines: A CleansedLines instance containing the file.
- linenum: The number of the line to check.
- error: The function to call with any errors found.
- """
- line = clean_lines.elided[linenum]
-
- # Avoid preprocessor lines
- if Match(r'^\s*#', line):
- return
-
- # Last ditch effort to avoid multi-line comments. This will not help
- # if the comment started before the current line or ended after the
- # current line, but it catches most of the false positives. At least,
- # it provides a way to workaround this warning for people who use
- # multi-line comments in preprocessor macros.
- #
- # TODO(unknown): remove this once cpplint has better support for
- # multi-line comments.
- if line.find('/*') >= 0 or line.find('*/') >= 0:
- return
-
- for match in _ALT_TOKEN_REPLACEMENT_PATTERN.finditer(line):
- error(filename, linenum, 'readability/alt_tokens', 2,
- 'Use operator %s instead of %s' % (
- _ALT_TOKEN_REPLACEMENT[match.group(1)], match.group(1)))
-
-
-def GetLineWidth(line):
- """Determines the width of the line in column positions.
-
- Args:
- line: A string, which may be a Unicode string.
-
- Returns:
- The width of the line in column positions, accounting for Unicode
- combining characters and wide characters.
- """
- if isinstance(line, unicode):
- width = 0
- for uc in unicodedata.normalize('NFC', line):
- if unicodedata.east_asian_width(uc) in ('W', 'F'):
- width += 2
- elif not unicodedata.combining(uc):
- width += 1
- return width
- else:
- return len(line)
-
-
-def CheckStyle(filename, clean_lines, linenum, file_extension, nesting_state,
- error):
- """Checks rules from the 'C++ style rules' section of cppguide.html.
-
- Most of these rules are hard to test (naming, comment style), but we
- do what we can. In particular we check for 2-space indents, line lengths,
- tab usage, spaces inside code, etc.
-
- Args:
- filename: The name of the current file.
- clean_lines: A CleansedLines instance containing the file.
- linenum: The number of the line to check.
- file_extension: The extension (without the dot) of the filename.
- nesting_state: A NestingState instance which maintains information about
- the current stack of nested blocks being parsed.
- error: The function to call with any errors found.
- """
-
- # Don't use "elided" lines here, otherwise we can't check commented lines.
- # Don't want to use "raw" either, because we don't want to check inside C++11
- # raw strings,
- raw_lines = clean_lines.lines_without_raw_strings
- line = raw_lines[linenum]
-
- if line.find('\t') != -1:
- error(filename, linenum, 'whitespace/tab', 1,
- 'Tab found; better to use spaces')
-
- # One or three blank spaces at the beginning of the line is weird; it's
- # hard to reconcile that with 2-space indents.
- # NOTE: here are the conditions rob pike used for his tests. Mine aren't
- # as sophisticated, but it may be worth becoming so: RLENGTH==initial_spaces
- # if(RLENGTH > 20) complain = 0;
- # if(match($0, " +(error|private|public|protected):")) complain = 0;
- # if(match(prev, "&& *$")) complain = 0;
- # if(match(prev, "\\|\\| *$")) complain = 0;
- # if(match(prev, "[\",=><] *$")) complain = 0;
- # if(match($0, " <<")) complain = 0;
- # if(match(prev, " +for \\(")) complain = 0;
- # if(prevodd && match(prevprev, " +for \\(")) complain = 0;
- scope_or_label_pattern = r'\s*\w+\s*:\s*\\?$'
- classinfo = nesting_state.InnermostClass()
- initial_spaces = 0
- cleansed_line = clean_lines.elided[linenum]
- while initial_spaces < len(line) and line[initial_spaces] == ' ':
- initial_spaces += 1
- if line and line[-1].isspace():
- error(filename, linenum, 'whitespace/end_of_line', 4,
- 'Line ends in whitespace. Consider deleting these extra spaces.')
- # There are certain situations we allow one space, notably for
- # section labels, and also lines containing multi-line raw strings.
- elif ((initial_spaces == 1 or initial_spaces == 3) and
- not Match(scope_or_label_pattern, cleansed_line) and
- not (clean_lines.raw_lines[linenum] != line and
- Match(r'^\s*""', line))):
- error(filename, linenum, 'whitespace/indent', 3,
- 'Weird number of spaces at line-start. '
- 'Are you using a 2-space indent?')
-
- # Check if the line is a header guard.
- is_header_guard = False
- if file_extension == 'h':
- cppvar = GetHeaderGuardCPPVariable(filename)
- if (line.startswith('#ifndef %s' % cppvar) or
- line.startswith('#define %s' % cppvar) or
- line.startswith('#endif // %s' % cppvar)):
- is_header_guard = True
- # #include lines and header guards can be long, since there's no clean way to
- # split them.
- #
- # URLs can be long too. It's possible to split these, but it makes them
- # harder to cut&paste.
- #
- # The "$Id:...$" comment may also get very long without it being the
- # developers fault.
- if (not line.startswith('#include') and not is_header_guard and
- not Match(r'^\s*//.*http(s?)://\S*$', line) and
- not Match(r'^// \$Id:.*#[0-9]+ \$$', line)):
- line_width = GetLineWidth(line)
- extended_length = int((_line_length * 1.25))
- if line_width > extended_length:
- error(filename, linenum, 'whitespace/line_length', 4,
- 'Lines should very rarely be longer than %i characters' %
- extended_length)
- elif line_width > _line_length:
- error(filename, linenum, 'whitespace/line_length', 2,
- 'Lines should be <= %i characters long' % _line_length)
-
- if (cleansed_line.count(';') > 1 and
- # for loops are allowed two ;'s (and may run over two lines).
- cleansed_line.find('for') == -1 and
- (GetPreviousNonBlankLine(clean_lines, linenum)[0].find('for') == -1 or
- GetPreviousNonBlankLine(clean_lines, linenum)[0].find(';') != -1) and
- # It's ok to have many commands in a switch case that fits in 1 line
- not ((cleansed_line.find('case ') != -1 or
- cleansed_line.find('default:') != -1) and
- cleansed_line.find('break;') != -1)):
- error(filename, linenum, 'whitespace/newline', 0,
- 'More than one command on the same line')
-
- # Some more style checks
- CheckBraces(filename, clean_lines, linenum, error)
- CheckTrailingSemicolon(filename, clean_lines, linenum, error)
- CheckEmptyBlockBody(filename, clean_lines, linenum, error)
- CheckAccess(filename, clean_lines, linenum, nesting_state, error)
- CheckSpacing(filename, clean_lines, linenum, nesting_state, error)
- CheckOperatorSpacing(filename, clean_lines, linenum, error)
- CheckParenthesisSpacing(filename, clean_lines, linenum, error)
- CheckCommaSpacing(filename, clean_lines, linenum, error)
- CheckBracesSpacing(filename, clean_lines, linenum, error)
- CheckSpacingForFunctionCall(filename, clean_lines, linenum, error)
- CheckRValueReference(filename, clean_lines, linenum, nesting_state, error)
- CheckCheck(filename, clean_lines, linenum, error)
- CheckAltTokens(filename, clean_lines, linenum, error)
- classinfo = nesting_state.InnermostClass()
- if classinfo:
- CheckSectionSpacing(filename, clean_lines, classinfo, linenum, error)
-
-
-_RE_PATTERN_INCLUDE = re.compile(r'^\s*#\s*include\s*([<"])([^>"]*)[>"].*$')
-# Matches the first component of a filename delimited by -s and _s. That is:
-# _RE_FIRST_COMPONENT.match('foo').group(0) == 'foo'
-# _RE_FIRST_COMPONENT.match('foo.cc').group(0) == 'foo'
-# _RE_FIRST_COMPONENT.match('foo-bar_baz.cc').group(0) == 'foo'
-# _RE_FIRST_COMPONENT.match('foo_bar-baz.cc').group(0) == 'foo'
-_RE_FIRST_COMPONENT = re.compile(r'^[^-_.]+')
-
-
-def _DropCommonSuffixes(filename):
- """Drops common suffixes like _test.cc or -inl.h from filename.
-
- For example:
- >>> _DropCommonSuffixes('foo/foo-inl.h')
- 'foo/foo'
- >>> _DropCommonSuffixes('foo/bar/foo.cc')
- 'foo/bar/foo'
- >>> _DropCommonSuffixes('foo/foo_internal.h')
- 'foo/foo'
- >>> _DropCommonSuffixes('foo/foo_unusualinternal.h')
- 'foo/foo_unusualinternal'
-
- Args:
- filename: The input filename.
-
- Returns:
- The filename with the common suffix removed.
- """
- for suffix in ('test.cc', 'regtest.cc', 'unittest.cc',
- 'inl.h', 'impl.h', 'internal.h'):
- if (filename.endswith(suffix) and len(filename) > len(suffix) and
- filename[-len(suffix) - 1] in ('-', '_')):
- return filename[:-len(suffix) - 1]
- return os.path.splitext(filename)[0]
-
-
-def _IsTestFilename(filename):
- """Determines if the given filename has a suffix that identifies it as a test.
-
- Args:
- filename: The input filename.
-
- Returns:
- True if 'filename' looks like a test, False otherwise.
- """
- if (filename.endswith('_test.cc') or
- filename.endswith('_unittest.cc') or
- filename.endswith('_regtest.cc')):
- return True
- else:
- return False
-
-
-def _ClassifyInclude(fileinfo, include, is_system):
- """Figures out what kind of header 'include' is.
-
- Args:
- fileinfo: The current file cpplint is running over. A FileInfo instance.
- include: The path to a #included file.
- is_system: True if the #include used <> rather than "".
-
- Returns:
- One of the _XXX_HEADER constants.
-
- For example:
- >>> _ClassifyInclude(FileInfo('foo/foo.cc'), 'stdio.h', True)
- _C_SYS_HEADER
- >>> _ClassifyInclude(FileInfo('foo/foo.cc'), 'string', True)
- _CPP_SYS_HEADER
- >>> _ClassifyInclude(FileInfo('foo/foo.cc'), 'foo/foo.h', False)
- _LIKELY_MY_HEADER
- >>> _ClassifyInclude(FileInfo('foo/foo_unknown_extension.cc'),
- ... 'bar/foo_other_ext.h', False)
- _POSSIBLE_MY_HEADER
- >>> _ClassifyInclude(FileInfo('foo/foo.cc'), 'foo/bar.h', False)
- _OTHER_HEADER
- """
- # This is a list of all standard c++ header files, except
- # those already checked for above.
- is_cpp_h = include in _CPP_HEADERS
-
- if is_system:
- if is_cpp_h:
- return _CPP_SYS_HEADER
- else:
- return _C_SYS_HEADER
-
- # If the target file and the include we're checking share a
- # basename when we drop common extensions, and the include
- # lives in . , then it's likely to be owned by the target file.
- target_dir, target_base = (
- os.path.split(_DropCommonSuffixes(fileinfo.RepositoryName())))
- include_dir, include_base = os.path.split(_DropCommonSuffixes(include))
- if target_base == include_base and (
- include_dir == target_dir or
- include_dir == os.path.normpath(target_dir + '/../public')):
- return _LIKELY_MY_HEADER
-
- # If the target and include share some initial basename
- # component, it's possible the target is implementing the
- # include, so it's allowed to be first, but we'll never
- # complain if it's not there.
- target_first_component = _RE_FIRST_COMPONENT.match(target_base)
- include_first_component = _RE_FIRST_COMPONENT.match(include_base)
- if (target_first_component and include_first_component and
- target_first_component.group(0) ==
- include_first_component.group(0)):
- return _POSSIBLE_MY_HEADER
-
- return _OTHER_HEADER
-
-
-
-def CheckIncludeLine(filename, clean_lines, linenum, include_state, error):
- """Check rules that are applicable to #include lines.
-
- Strings on #include lines are NOT removed from elided line, to make
- certain tasks easier. However, to prevent false positives, checks
- applicable to #include lines in CheckLanguage must be put here.
-
- Args:
- filename: The name of the current file.
- clean_lines: A CleansedLines instance containing the file.
- linenum: The number of the line to check.
- include_state: An _IncludeState instance in which the headers are inserted.
- error: The function to call with any errors found.
- """
- fileinfo = FileInfo(filename)
- line = clean_lines.lines[linenum]
-
- # "include" should use the new style "foo/bar.h" instead of just "bar.h"
- # Only do this check if the included header follows google naming
- # conventions. If not, assume that it's a 3rd party API that
- # requires special include conventions.
- #
- # We also make an exception for Lua headers, which follow google
- # naming convention but not the include convention.
- match = Match(r'#include\s*"([^/]+\.h)"', line)
- if match and not _THIRD_PARTY_HEADERS_PATTERN.match(match.group(1)):
- error(filename, linenum, 'build/include', 4,
- 'Include the directory when naming .h files')
-
- # we shouldn't include a file more than once. actually, there are a
- # handful of instances where doing so is okay, but in general it's
- # not.
- match = _RE_PATTERN_INCLUDE.search(line)
- if match:
- include = match.group(2)
- is_system = (match.group(1) == '<')
- duplicate_line = include_state.FindHeader(include)
- if duplicate_line >= 0:
- error(filename, linenum, 'build/include', 4,
- '"%s" already included at %s:%s' %
- (include, filename, duplicate_line))
- elif (include.endswith('.cc') and
- os.path.dirname(fileinfo.RepositoryName()) != os.path.dirname(include)):
- error(filename, linenum, 'build/include', 4,
- 'Do not include .cc files from other packages')
- elif not _THIRD_PARTY_HEADERS_PATTERN.match(include):
- include_state.include_list[-1].append((include, linenum))
-
- # We want to ensure that headers appear in the right order:
- # 1) for foo.cc, foo.h (preferred location)
- # 2) c system files
- # 3) cpp system files
- # 4) for foo.cc, foo.h (deprecated location)
- # 5) other google headers
- #
- # We classify each include statement as one of those 5 types
- # using a number of techniques. The include_state object keeps
- # track of the highest type seen, and complains if we see a
- # lower type after that.
- error_message = include_state.CheckNextIncludeOrder(
- _ClassifyInclude(fileinfo, include, is_system))
- if error_message:
- error(filename, linenum, 'build/include_order', 4,
- '%s. Should be: %s.h, c system, c++ system, other.' %
- (error_message, fileinfo.BaseName()))
- canonical_include = include_state.CanonicalizeAlphabeticalOrder(include)
- if not include_state.IsInAlphabeticalOrder(
- clean_lines, linenum, canonical_include):
- error(filename, linenum, 'build/include_alpha', 4,
- 'Include "%s" not in alphabetical order' % include)
- include_state.SetLastHeader(canonical_include)
-
-
-
-def _GetTextInside(text, start_pattern):
- r"""Retrieves all the text between matching open and close parentheses.
-
- Given a string of lines and a regular expression string, retrieve all the text
- following the expression and between opening punctuation symbols like
- (, [, or {, and the matching close-punctuation symbol. This properly nested
- occurrences of the punctuations, so for the text like
- printf(a(), b(c()));
- a call to _GetTextInside(text, r'printf\(') will return 'a(), b(c())'.
- start_pattern must match string having an open punctuation symbol at the end.
-
- Args:
- text: The lines to extract text. Its comments and strings must be elided.
- It can be single line and can span multiple lines.
- start_pattern: The regexp string indicating where to start extracting
- the text.
- Returns:
- The extracted text.
- None if either the opening string or ending punctuation could not be found.
- """
- # TODO(unknown): Audit cpplint.py to see what places could be profitably
- # rewritten to use _GetTextInside (and use inferior regexp matching today).
-
- # Give opening punctuations to get the matching close-punctuations.
- matching_punctuation = {'(': ')', '{': '}', '[': ']'}
- closing_punctuation = set(matching_punctuation.itervalues())
-
- # Find the position to start extracting text.
- match = re.search(start_pattern, text, re.M)
- if not match: # start_pattern not found in text.
- return None
- start_position = match.end(0)
-
- assert start_position > 0, (
- 'start_pattern must ends with an opening punctuation.')
- assert text[start_position - 1] in matching_punctuation, (
- 'start_pattern must ends with an opening punctuation.')
- # Stack of closing punctuations we expect to have in text after position.
- punctuation_stack = [matching_punctuation[text[start_position - 1]]]
- position = start_position
- while punctuation_stack and position < len(text):
- if text[position] == punctuation_stack[-1]:
- punctuation_stack.pop()
- elif text[position] in closing_punctuation:
- # A closing punctuation without matching opening punctuations.
- return None
- elif text[position] in matching_punctuation:
- punctuation_stack.append(matching_punctuation[text[position]])
- position += 1
- if punctuation_stack:
- # Opening punctuations left without matching close-punctuations.
- return None
- # punctuations match.
- return text[start_position:position - 1]
-
-
-# Patterns for matching call-by-reference parameters.
-#
-# Supports nested templates up to 2 levels deep using this messy pattern:
-# < (?: < (?: < [^<>]*
-# >
-# | [^<>] )*
-# >
-# | [^<>] )*
-# >
-_RE_PATTERN_IDENT = r'[_a-zA-Z]\w*' # =~ [[:alpha:]][[:alnum:]]*
-_RE_PATTERN_TYPE = (
- r'(?:const\s+)?(?:typename\s+|class\s+|struct\s+|union\s+|enum\s+)?'
- r'(?:\w|'
- r'\s*<(?:<(?:<[^<>]*>|[^<>])*>|[^<>])*>|'
- r'::)+')
-# A call-by-reference parameter ends with '& identifier'.
-_RE_PATTERN_REF_PARAM = re.compile(
- r'(' + _RE_PATTERN_TYPE + r'(?:\s*(?:\bconst\b|[*]))*\s*'
- r'&\s*' + _RE_PATTERN_IDENT + r')\s*(?:=[^,()]+)?[,)]')
-# A call-by-const-reference parameter either ends with 'const& identifier'
-# or looks like 'const type& identifier' when 'type' is atomic.
-_RE_PATTERN_CONST_REF_PARAM = (
- r'(?:.*\s*\bconst\s*&\s*' + _RE_PATTERN_IDENT +
- r'|const\s+' + _RE_PATTERN_TYPE + r'\s*&\s*' + _RE_PATTERN_IDENT + r')')
-
-
-def CheckLanguage(filename, clean_lines, linenum, file_extension,
- include_state, nesting_state, error):
- """Checks rules from the 'C++ language rules' section of cppguide.html.
-
- Some of these rules are hard to test (function overloading, using
- uint32 inappropriately), but we do the best we can.
-
- Args:
- filename: The name of the current file.
- clean_lines: A CleansedLines instance containing the file.
- linenum: The number of the line to check.
- file_extension: The extension (without the dot) of the filename.
- include_state: An _IncludeState instance in which the headers are inserted.
- nesting_state: A NestingState instance which maintains information about
- the current stack of nested blocks being parsed.
- error: The function to call with any errors found.
- """
- # If the line is empty or consists of entirely a comment, no need to
- # check it.
- line = clean_lines.elided[linenum]
- if not line:
- return
-
- match = _RE_PATTERN_INCLUDE.search(line)
- if match:
- CheckIncludeLine(filename, clean_lines, linenum, include_state, error)
- return
-
- # Reset include state across preprocessor directives. This is meant
- # to silence warnings for conditional includes.
- match = Match(r'^\s*#\s*(if|ifdef|ifndef|elif|else|endif)\b', line)
- if match:
- include_state.ResetSection(match.group(1))
-
- # Make Windows paths like Unix.
- fullname = os.path.abspath(filename).replace('\\', '/')
-
- # Perform other checks now that we are sure that this is not an include line
- CheckCasts(filename, clean_lines, linenum, error)
- CheckGlobalStatic(filename, clean_lines, linenum, error)
- CheckPrintf(filename, clean_lines, linenum, error)
-
- if file_extension == 'h':
- # TODO(unknown): check that 1-arg constructors are explicit.
- # How to tell it's a constructor?
- # (handled in CheckForNonStandardConstructs for now)
- # TODO(unknown): check that classes declare or disable copy/assign
- # (level 1 error)
- pass
-
- # Check if people are using the verboten C basic types. The only exception
- # we regularly allow is "unsigned short port" for port.
- if Search(r'\bshort port\b', line):
- if not Search(r'\bunsigned short port\b', line):
- error(filename, linenum, 'runtime/int', 4,
- 'Use "unsigned short" for ports, not "short"')
- else:
- match = Search(r'\b(short|long(?! +double)|long long)\b', line)
- if match:
- error(filename, linenum, 'runtime/int', 4,
- 'Use int16/int64/etc, rather than the C type %s' % match.group(1))
-
- # Check if some verboten operator overloading is going on
- # TODO(unknown): catch out-of-line unary operator&:
- # class X {};
- # int operator&(const X& x) { return 42; } // unary operator&
- # The trick is it's hard to tell apart from binary operator&:
- # class Y { int operator&(const Y& x) { return 23; } }; // binary operator&
- if Search(r'\boperator\s*&\s*\(\s*\)', line):
- error(filename, linenum, 'runtime/operator', 4,
- 'Unary operator& is dangerous. Do not use it.')
-
- # Check for suspicious usage of "if" like
- # } if (a == b) {
- if Search(r'\}\s*if\s*\(', line):
- error(filename, linenum, 'readability/braces', 4,
- 'Did you mean "else if"? If not, start a new line for "if".')
-
- # Check for potential format string bugs like printf(foo).
- # We constrain the pattern not to pick things like DocidForPrintf(foo).
- # Not perfect but it can catch printf(foo.c_str()) and printf(foo->c_str())
- # TODO(unknown): Catch the following case. Need to change the calling
- # convention of the whole function to process multiple line to handle it.
- # printf(
- # boy_this_is_a_really_long_variable_that_cannot_fit_on_the_prev_line);
- printf_args = _GetTextInside(line, r'(?i)\b(string)?printf\s*\(')
- if printf_args:
- match = Match(r'([\w.\->()]+)$', printf_args)
- if match and match.group(1) != '__VA_ARGS__':
- function_name = re.search(r'\b((?:string)?printf)\s*\(',
- line, re.I).group(1)
- error(filename, linenum, 'runtime/printf', 4,
- 'Potential format string bug. Do %s("%%s", %s) instead.'
- % (function_name, match.group(1)))
-
- # Check for potential memset bugs like memset(buf, sizeof(buf), 0).
- match = Search(r'memset\s*\(([^,]*),\s*([^,]*),\s*0\s*\)', line)
- if match and not Match(r"^''|-?[0-9]+|0x[0-9A-Fa-f]$", match.group(2)):
- error(filename, linenum, 'runtime/memset', 4,
- 'Did you mean "memset(%s, 0, %s)"?'
- % (match.group(1), match.group(2)))
-
- if Search(r'\busing namespace\b', line):
- error(filename, linenum, 'build/namespaces', 5,
- 'Do not use namespace using-directives. '
- 'Use using-declarations instead.')
-
- # Detect variable-length arrays.
- match = Match(r'\s*(.+::)?(\w+) [a-z]\w*\[(.+)];', line)
- if (match and match.group(2) != 'return' and match.group(2) != 'delete' and
- match.group(3).find(']') == -1):
- # Split the size using space and arithmetic operators as delimiters.
- # If any of the resulting tokens are not compile time constants then
- # report the error.
- tokens = re.split(r'\s|\+|\-|\*|\/|<<|>>]', match.group(3))
- is_const = True
- skip_next = False
- for tok in tokens:
- if skip_next:
- skip_next = False
- continue
-
- if Search(r'sizeof\(.+\)', tok): continue
- if Search(r'arraysize\(\w+\)', tok): continue
-
- tok = tok.lstrip('(')
- tok = tok.rstrip(')')
- if not tok: continue
- if Match(r'\d+', tok): continue
- if Match(r'0[xX][0-9a-fA-F]+', tok): continue
- if Match(r'k[A-Z0-9]\w*', tok): continue
- if Match(r'(.+::)?k[A-Z0-9]\w*', tok): continue
- if Match(r'(.+::)?[A-Z][A-Z0-9_]*', tok): continue
- # A catch all for tricky sizeof cases, including 'sizeof expression',
- # 'sizeof(*type)', 'sizeof(const type)', 'sizeof(struct StructName)'
- # requires skipping the next token because we split on ' ' and '*'.
- if tok.startswith('sizeof'):
- skip_next = True
- continue
- is_const = False
- break
- if not is_const:
- error(filename, linenum, 'runtime/arrays', 1,
- 'Do not use variable-length arrays. Use an appropriately named '
- "('k' followed by CamelCase) compile-time constant for the size.")
-
- # Check for use of unnamed namespaces in header files. Registration
- # macros are typically OK, so we allow use of "namespace {" on lines
- # that end with backslashes.
- if (file_extension == 'h'
- and Search(r'\bnamespace\s*{', line)
- and line[-1] != '\\'):
- error(filename, linenum, 'build/namespaces', 4,
- 'Do not use unnamed namespaces in header files. See '
- 'http://google-styleguide.googlecode.com/svn/trunk/cppguide.xml#Namespaces'
- ' for more information.')
-
-
-def CheckGlobalStatic(filename, clean_lines, linenum, error):
- """Check for unsafe global or static objects.
-
- Args:
- filename: The name of the current file.
- clean_lines: A CleansedLines instance containing the file.
- linenum: The number of the line to check.
- error: The function to call with any errors found.
- """
- line = clean_lines.elided[linenum]
-
- # Match two lines at a time to support multiline declarations
- if linenum + 1 < clean_lines.NumLines() and not Search(r'[;({]', line):
- line += clean_lines.elided[linenum + 1].strip()
-
- # Check for people declaring static/global STL strings at the top level.
- # This is dangerous because the C++ language does not guarantee that
- # globals with constructors are initialized before the first access.
- match = Match(
- r'((?:|static +)(?:|const +))string +([a-zA-Z0-9_:]+)\b(.*)',
- line)
-
- # Remove false positives:
- # - String pointers (as opposed to values).
- # string *pointer
- # const string *pointer
- # string const *pointer
- # string *const pointer
- #
- # - Functions and template specializations.
- # string Function<Type>(...
- # string Class<Type>::Method(...
- #
- # - Operators. These are matched separately because operator names
- # cross non-word boundaries, and trying to match both operators
- # and functions at the same time would decrease accuracy of
- # matching identifiers.
- # string Class::operator*()
- if (match and
- not Search(r'\bstring\b(\s+const)?\s*\*\s*(const\s+)?\w', line) and
- not Search(r'\boperator\W', line) and
- not Match(r'\s*(<.*>)?(::[a-zA-Z0-9_]+)*\s*\(([^"]|$)', match.group(3))):
- error(filename, linenum, 'runtime/string', 4,
- 'For a static/global string constant, use a C style string instead: '
- '"%schar %s[]".' %
- (match.group(1), match.group(2)))
-
- if Search(r'\b([A-Za-z0-9_]*_)\(\1\)', line):
- error(filename, linenum, 'runtime/init', 4,
- 'You seem to be initializing a member variable with itself.')
-
-
-def CheckPrintf(filename, clean_lines, linenum, error):
- """Check for printf related issues.
-
- Args:
- filename: The name of the current file.
- clean_lines: A CleansedLines instance containing the file.
- linenum: The number of the line to check.
- error: The function to call with any errors found.
- """
- line = clean_lines.elided[linenum]
-
- # When snprintf is used, the second argument shouldn't be a literal.
- match = Search(r'snprintf\s*\(([^,]*),\s*([0-9]*)\s*,', line)
- if match and match.group(2) != '0':
- # If 2nd arg is zero, snprintf is used to calculate size.
- error(filename, linenum, 'runtime/printf', 3,
- 'If you can, use sizeof(%s) instead of %s as the 2nd arg '
- 'to snprintf.' % (match.group(1), match.group(2)))
-
- # Check if some verboten C functions are being used.
- if Search(r'\bsprintf\s*\(', line):
- error(filename, linenum, 'runtime/printf', 5,
- 'Never use sprintf. Use snprintf instead.')
- match = Search(r'\b(strcpy|strcat)\s*\(', line)
- if match:
- error(filename, linenum, 'runtime/printf', 4,
- 'Almost always, snprintf is better than %s' % match.group(1))
-
-
-def IsDerivedFunction(clean_lines, linenum):
- """Check if current line contains an inherited function.
-
- Args:
- clean_lines: A CleansedLines instance containing the file.
- linenum: The number of the line to check.
- Returns:
- True if current line contains a function with "override"
- virt-specifier.
- """
- # Scan back a few lines for start of current function
- for i in xrange(linenum, max(-1, linenum - 10), -1):
- match = Match(r'^([^()]*\w+)\(', clean_lines.elided[i])
- if match:
- # Look for "override" after the matching closing parenthesis
- line, _, closing_paren = CloseExpression(
- clean_lines, i, len(match.group(1)))
- return (closing_paren >= 0 and
- Search(r'\boverride\b', line[closing_paren:]))
- return False
-
-
-def IsOutOfLineMethodDefinition(clean_lines, linenum):
- """Check if current line contains an out-of-line method definition.
-
- Args:
- clean_lines: A CleansedLines instance containing the file.
- linenum: The number of the line to check.
- Returns:
- True if current line contains an out-of-line method definition.
- """
- # Scan back a few lines for start of current function
- for i in xrange(linenum, max(-1, linenum - 10), -1):
- if Match(r'^([^()]*\w+)\(', clean_lines.elided[i]):
- return Match(r'^[^()]*\w+::\w+\(', clean_lines.elided[i]) is not None
- return False
-
-
-def IsInitializerList(clean_lines, linenum):
- """Check if current line is inside constructor initializer list.
-
- Args:
- clean_lines: A CleansedLines instance containing the file.
- linenum: The number of the line to check.
- Returns:
- True if current line appears to be inside constructor initializer
- list, False otherwise.
- """
- for i in xrange(linenum, 1, -1):
- line = clean_lines.elided[i]
- if i == linenum:
- remove_function_body = Match(r'^(.*)\{\s*$', line)
- if remove_function_body:
- line = remove_function_body.group(1)
-
- if Search(r'\s:\s*\w+[({]', line):
- # A lone colon tend to indicate the start of a constructor
- # initializer list. It could also be a ternary operator, which
- # also tend to appear in constructor initializer lists as
- # opposed to parameter lists.
- return True
- if Search(r'\}\s*,\s*$', line):
- # A closing brace followed by a comma is probably the end of a
- # brace-initialized member in constructor initializer list.
- return True
- if Search(r'[{};]\s*$', line):
- # Found one of the following:
- # - A closing brace or semicolon, probably the end of the previous
- # function.
- # - An opening brace, probably the start of current class or namespace.
- #
- # Current line is probably not inside an initializer list since
- # we saw one of those things without seeing the starting colon.
- return False
-
- # Got to the beginning of the file without seeing the start of
- # constructor initializer list.
- return False
-
-
-def CheckForNonConstReference(filename, clean_lines, linenum,
- nesting_state, error):
- """Check for non-const references.
-
- Separate from CheckLanguage since it scans backwards from current
- line, instead of scanning forward.
-
- Args:
- filename: The name of the current file.
- clean_lines: A CleansedLines instance containing the file.
- linenum: The number of the line to check.
- nesting_state: A NestingState instance which maintains information about
- the current stack of nested blocks being parsed.
- error: The function to call with any errors found.
- """
- # Do nothing if there is no '&' on current line.
- line = clean_lines.elided[linenum]
- if '&' not in line:
- return
-
- # If a function is inherited, current function doesn't have much of
- # a choice, so any non-const references should not be blamed on
- # derived function.
- if IsDerivedFunction(clean_lines, linenum):
- return
-
- # Don't warn on out-of-line method definitions, as we would warn on the
- # in-line declaration, if it isn't marked with 'override'.
- if IsOutOfLineMethodDefinition(clean_lines, linenum):
- return
-
- # Long type names may be broken across multiple lines, usually in one
- # of these forms:
- # LongType
- # ::LongTypeContinued &identifier
- # LongType::
- # LongTypeContinued &identifier
- # LongType<
- # ...>::LongTypeContinued &identifier
- #
- # If we detected a type split across two lines, join the previous
- # line to current line so that we can match const references
- # accordingly.
- #
- # Note that this only scans back one line, since scanning back
- # arbitrary number of lines would be expensive. If you have a type
- # that spans more than 2 lines, please use a typedef.
- if linenum > 1:
- previous = None
- if Match(r'\s*::(?:[\w<>]|::)+\s*&\s*\S', line):
- # previous_line\n + ::current_line
- previous = Search(r'\b((?:const\s*)?(?:[\w<>]|::)+[\w<>])\s*$',
- clean_lines.elided[linenum - 1])
- elif Match(r'\s*[a-zA-Z_]([\w<>]|::)+\s*&\s*\S', line):
- # previous_line::\n + current_line
- previous = Search(r'\b((?:const\s*)?(?:[\w<>]|::)+::)\s*$',
- clean_lines.elided[linenum - 1])
- if previous:
- line = previous.group(1) + line.lstrip()
- else:
- # Check for templated parameter that is split across multiple lines
- endpos = line.rfind('>')
- if endpos > -1:
- (_, startline, startpos) = ReverseCloseExpression(
- clean_lines, linenum, endpos)
- if startpos > -1 and startline < linenum:
- # Found the matching < on an earlier line, collect all
- # pieces up to current line.
- line = ''
- for i in xrange(startline, linenum + 1):
- line += clean_lines.elided[i].strip()
-
- # Check for non-const references in function parameters. A single '&' may
- # found in the following places:
- # inside expression: binary & for bitwise AND
- # inside expression: unary & for taking the address of something
- # inside declarators: reference parameter
- # We will exclude the first two cases by checking that we are not inside a
- # function body, including one that was just introduced by a trailing '{'.
- # TODO(unknown): Doesn't account for 'catch(Exception& e)' [rare].
- if (nesting_state.previous_stack_top and
- not (isinstance(nesting_state.previous_stack_top, _ClassInfo) or
- isinstance(nesting_state.previous_stack_top, _NamespaceInfo))):
- # Not at toplevel, not within a class, and not within a namespace
- return
-
- # Avoid initializer lists. We only need to scan back from the
- # current line for something that starts with ':'.
- #
- # We don't need to check the current line, since the '&' would
- # appear inside the second set of parentheses on the current line as
- # opposed to the first set.
- if linenum > 0:
- for i in xrange(linenum - 1, max(0, linenum - 10), -1):
- previous_line = clean_lines.elided[i]
- if not Search(r'[),]\s*$', previous_line):
- break
- if Match(r'^\s*:\s+\S', previous_line):
- return
-
- # Avoid preprocessors
- if Search(r'\\\s*$', line):
- return
-
- # Avoid constructor initializer lists
- if IsInitializerList(clean_lines, linenum):
- return
-
- # We allow non-const references in a few standard places, like functions
- # called "swap()" or iostream operators like "<<" or ">>". Do not check
- # those function parameters.
- #
- # We also accept & in static_assert, which looks like a function but
- # it's actually a declaration expression.
- whitelisted_functions = (r'(?:[sS]wap(?:<\w:+>)?|'
- r'operator\s*[<>][<>]|'
- r'static_assert|COMPILE_ASSERT'
- r')\s*\(')
- if Search(whitelisted_functions, line):
- return
- elif not Search(r'\S+\([^)]*$', line):
- # Don't see a whitelisted function on this line. Actually we
- # didn't see any function name on this line, so this is likely a
- # multi-line parameter list. Try a bit harder to catch this case.
- for i in xrange(2):
- if (linenum > i and
- Search(whitelisted_functions, clean_lines.elided[linenum - i - 1])):
- return
-
- decls = ReplaceAll(r'{[^}]*}', ' ', line) # exclude function body
- for parameter in re.findall(_RE_PATTERN_REF_PARAM, decls):
- if not Match(_RE_PATTERN_CONST_REF_PARAM, parameter):
- error(filename, linenum, 'runtime/references', 2,
- 'Is this a non-const reference? '
- 'If so, make const or use a pointer: ' +
- ReplaceAll(' *<', '<', parameter))
-
-
-def CheckCasts(filename, clean_lines, linenum, error):
- """Various cast related checks.
-
- Args:
- filename: The name of the current file.
- clean_lines: A CleansedLines instance containing the file.
- linenum: The number of the line to check.
- error: The function to call with any errors found.
- """
- line = clean_lines.elided[linenum]
-
- # Check to see if they're using an conversion function cast.
- # I just try to capture the most common basic types, though there are more.
- # Parameterless conversion functions, such as bool(), are allowed as they are
- # probably a member operator declaration or default constructor.
- match = Search(
- r'(\bnew\s+|\S<\s*(?:const\s+)?)?\b'
- r'(int|float|double|bool|char|int32|uint32|int64|uint64)'
- r'(\([^)].*)', line)
- expecting_function = ExpectingFunctionArgs(clean_lines, linenum)
- if match and not expecting_function:
- matched_type = match.group(2)
-
- # matched_new_or_template is used to silence two false positives:
- # - New operators
- # - Template arguments with function types
- #
- # For template arguments, we match on types immediately following
- # an opening bracket without any spaces. This is a fast way to
- # silence the common case where the function type is the first
- # template argument. False negative with less-than comparison is
- # avoided because those operators are usually followed by a space.
- #
- # function<double(double)> // bracket + no space = false positive
- # value < double(42) // bracket + space = true positive
- matched_new_or_template = match.group(1)
-
- # Avoid arrays by looking for brackets that come after the closing
- # parenthesis.
- if Match(r'\([^()]+\)\s*\[', match.group(3)):
- return
-
- # Other things to ignore:
- # - Function pointers
- # - Casts to pointer types
- # - Placement new
- # - Alias declarations
- matched_funcptr = match.group(3)
- if (matched_new_or_template is None and
- not (matched_funcptr and
- (Match(r'\((?:[^() ]+::\s*\*\s*)?[^() ]+\)\s*\(',
- matched_funcptr) or
- matched_funcptr.startswith('(*)'))) and
- not Match(r'\s*using\s+\S+\s*=\s*' + matched_type, line) and
- not Search(r'new\(\S+\)\s*' + matched_type, line)):
- error(filename, linenum, 'readability/casting', 4,
- 'Using deprecated casting style. '
- 'Use static_cast<%s>(...) instead' %
- matched_type)
-
- if not expecting_function:
- CheckCStyleCast(filename, clean_lines, linenum, 'static_cast',
- r'\((int|float|double|bool|char|u?int(16|32|64))\)', error)
-
- # This doesn't catch all cases. Consider (const char * const)"hello".
- #
- # (char *) "foo" should always be a const_cast (reinterpret_cast won't
- # compile).
- if CheckCStyleCast(filename, clean_lines, linenum, 'const_cast',
- r'\((char\s?\*+\s?)\)\s*"', error):
- pass
- else:
- # Check pointer casts for other than string constants
- CheckCStyleCast(filename, clean_lines, linenum, 'reinterpret_cast',
- r'\((\w+\s?\*+\s?)\)', error)
-
- # In addition, we look for people taking the address of a cast. This
- # is dangerous -- casts can assign to temporaries, so the pointer doesn't
- # point where you think.
- #
- # Some non-identifier character is required before the '&' for the
- # expression to be recognized as a cast. These are casts:
- # expression = &static_cast<int*>(temporary());
- # function(&(int*)(temporary()));
- #
- # This is not a cast:
- # reference_type&(int* function_param);
- match = Search(
- r'(?:[^\w]&\(([^)*][^)]*)\)[\w(])|'
- r'(?:[^\w]&(static|dynamic|down|reinterpret)_cast\b)', line)
- if match:
- # Try a better error message when the & is bound to something
- # dereferenced by the casted pointer, as opposed to the casted
- # pointer itself.
- parenthesis_error = False
- match = Match(r'^(.*&(?:static|dynamic|down|reinterpret)_cast\b)<', line)
- if match:
- _, y1, x1 = CloseExpression(clean_lines, linenum, len(match.group(1)))
- if x1 >= 0 and clean_lines.elided[y1][x1] == '(':
- _, y2, x2 = CloseExpression(clean_lines, y1, x1)
- if x2 >= 0:
- extended_line = clean_lines.elided[y2][x2:]
- if y2 < clean_lines.NumLines() - 1:
- extended_line += clean_lines.elided[y2 + 1]
- if Match(r'\s*(?:->|\[)', extended_line):
- parenthesis_error = True
-
- if parenthesis_error:
- error(filename, linenum, 'readability/casting', 4,
- ('Are you taking an address of something dereferenced '
- 'from a cast? Wrapping the dereferenced expression in '
- 'parentheses will make the binding more obvious'))
- else:
- error(filename, linenum, 'runtime/casting', 4,
- ('Are you taking an address of a cast? '
- 'This is dangerous: could be a temp var. '
- 'Take the address before doing the cast, rather than after'))
-
-
-def CheckCStyleCast(filename, clean_lines, linenum, cast_type, pattern, error):
- """Checks for a C-style cast by looking for the pattern.
-
- Args:
- filename: The name of the current file.
- clean_lines: A CleansedLines instance containing the file.
- linenum: The number of the line to check.
- cast_type: The string for the C++ cast to recommend. This is either
- reinterpret_cast, static_cast, or const_cast, depending.
- pattern: The regular expression used to find C-style casts.
- error: The function to call with any errors found.
-
- Returns:
- True if an error was emitted.
- False otherwise.
- """
- line = clean_lines.elided[linenum]
- match = Search(pattern, line)
- if not match:
- return False
-
- # Exclude lines with keywords that tend to look like casts
- context = line[0:match.start(1) - 1]
- if Match(r'.*\b(?:sizeof|alignof|alignas|[_A-Z][_A-Z0-9]*)\s*$', context):
- return False
-
- # Try expanding current context to see if we one level of
- # parentheses inside a macro.
- if linenum > 0:
- for i in xrange(linenum - 1, max(0, linenum - 5), -1):
- context = clean_lines.elided[i] + context
- if Match(r'.*\b[_A-Z][_A-Z0-9]*\s*\((?:\([^()]*\)|[^()])*$', context):
- return False
-
- # operator++(int) and operator--(int)
- if context.endswith(' operator++') or context.endswith(' operator--'):
- return False
-
- # A single unnamed argument for a function tends to look like old
- # style cast. If we see those, don't issue warnings for deprecated
- # casts, instead issue warnings for unnamed arguments where
- # appropriate.
- #
- # These are things that we want warnings for, since the style guide
- # explicitly require all parameters to be named:
- # Function(int);
- # Function(int) {
- # ConstMember(int) const;
- # ConstMember(int) const {
- # ExceptionMember(int) throw (...);
- # ExceptionMember(int) throw (...) {
- # PureVirtual(int) = 0;
- # [](int) -> bool {
- #
- # These are functions of some sort, where the compiler would be fine
- # if they had named parameters, but people often omit those
- # identifiers to reduce clutter:
- # (FunctionPointer)(int);
- # (FunctionPointer)(int) = value;
- # Function((function_pointer_arg)(int))
- # Function((function_pointer_arg)(int), int param)
- # <TemplateArgument(int)>;
- # <(FunctionPointerTemplateArgument)(int)>;
- remainder = line[match.end(0):]
- if Match(r'^\s*(?:;|const\b|throw\b|final\b|override\b|[=>{),]|->)',
- remainder):
- # Looks like an unnamed parameter.
-
- # Don't warn on any kind of template arguments.
- if Match(r'^\s*>', remainder):
- return False
-
- # Don't warn on assignments to function pointers, but keep warnings for
- # unnamed parameters to pure virtual functions. Note that this pattern
- # will also pass on assignments of "0" to function pointers, but the
- # preferred values for those would be "nullptr" or "NULL".
- matched_zero = Match(r'^\s=\s*(\S+)\s*;', remainder)
- if matched_zero and matched_zero.group(1) != '0':
- return False
-
- # Don't warn on function pointer declarations. For this we need
- # to check what came before the "(type)" string.
- if Match(r'.*\)\s*$', line[0:match.start(0)]):
- return False
-
- # Don't warn if the parameter is named with block comments, e.g.:
- # Function(int /*unused_param*/);
- raw_line = clean_lines.raw_lines[linenum]
- if '/*' in raw_line:
- return False
-
- # Passed all filters, issue warning here.
- error(filename, linenum, 'readability/function', 3,
- 'All parameters should be named in a function')
- return True
-
- # At this point, all that should be left is actual casts.
- error(filename, linenum, 'readability/casting', 4,
- 'Using C-style cast. Use %s<%s>(...) instead' %
- (cast_type, match.group(1)))
-
- return True
-
-
-def ExpectingFunctionArgs(clean_lines, linenum):
- """Checks whether where function type arguments are expected.
-
- Args:
- clean_lines: A CleansedLines instance containing the file.
- linenum: The number of the line to check.
-
- Returns:
- True if the line at 'linenum' is inside something that expects arguments
- of function types.
- """
- line = clean_lines.elided[linenum]
- return (Match(r'^\s*MOCK_(CONST_)?METHOD\d+(_T)?\(', line) or
- (linenum >= 2 and
- (Match(r'^\s*MOCK_(?:CONST_)?METHOD\d+(?:_T)?\((?:\S+,)?\s*$',
- clean_lines.elided[linenum - 1]) or
- Match(r'^\s*MOCK_(?:CONST_)?METHOD\d+(?:_T)?\(\s*$',
- clean_lines.elided[linenum - 2]) or
- Search(r'\bstd::m?function\s*\<\s*$',
- clean_lines.elided[linenum - 1]))))
-
-
-_HEADERS_CONTAINING_TEMPLATES = (
- ('<deque>', ('deque',)),
- ('<functional>', ('unary_function', 'binary_function',
- 'plus', 'minus', 'multiplies', 'divides', 'modulus',
- 'negate',
- 'equal_to', 'not_equal_to', 'greater', 'less',
- 'greater_equal', 'less_equal',
- 'logical_and', 'logical_or', 'logical_not',
- 'unary_negate', 'not1', 'binary_negate', 'not2',
- 'bind1st', 'bind2nd',
- 'pointer_to_unary_function',
- 'pointer_to_binary_function',
- 'ptr_fun',
- 'mem_fun_t', 'mem_fun', 'mem_fun1_t', 'mem_fun1_ref_t',
- 'mem_fun_ref_t',
- 'const_mem_fun_t', 'const_mem_fun1_t',
- 'const_mem_fun_ref_t', 'const_mem_fun1_ref_t',
- 'mem_fun_ref',
- )),
- ('<limits>', ('numeric_limits',)),
- ('<list>', ('list',)),
- ('<map>', ('map', 'multimap',)),
- ('<memory>', ('allocator',)),
- ('<queue>', ('queue', 'priority_queue',)),
- ('<set>', ('set', 'multiset',)),
- ('<stack>', ('stack',)),
- ('<string>', ('char_traits', 'basic_string',)),
- ('<tuple>', ('tuple',)),
- ('<utility>', ('pair',)),
- ('<vector>', ('vector',)),
-
- # gcc extensions.
- # Note: std::hash is their hash, ::hash is our hash
- ('<hash_map>', ('hash_map', 'hash_multimap',)),
- ('<hash_set>', ('hash_set', 'hash_multiset',)),
- ('<slist>', ('slist',)),
- )
-
-_RE_PATTERN_STRING = re.compile(r'\bstring\b')
-
-_re_pattern_algorithm_header = []
-for _template in ('copy', 'max', 'min', 'min_element', 'sort', 'swap',
- 'transform'):
- # Match max<type>(..., ...), max(..., ...), but not foo->max, foo.max or
- # type::max().
- _re_pattern_algorithm_header.append(
- (re.compile(r'[^>.]\b' + _template + r'(<.*?>)?\([^\)]'),
- _template,
- '<algorithm>'))
-
-_re_pattern_templates = []
-for _header, _templates in _HEADERS_CONTAINING_TEMPLATES:
- for _template in _templates:
- _re_pattern_templates.append(
- (re.compile(r'(\<|\b)' + _template + r'\s*\<'),
- _template + '<>',
- _header))
-
-
-def FilesBelongToSameModule(filename_cc, filename_h):
- """Check if these two filenames belong to the same module.
-
- The concept of a 'module' here is a as follows:
- foo.h, foo-inl.h, foo.cc, foo_test.cc and foo_unittest.cc belong to the
- same 'module' if they are in the same directory.
- some/path/public/xyzzy and some/path/internal/xyzzy are also considered
- to belong to the same module here.
-
- If the filename_cc contains a longer path than the filename_h, for example,
- '/absolute/path/to/base/sysinfo.cc', and this file would include
- 'base/sysinfo.h', this function also produces the prefix needed to open the
- header. This is used by the caller of this function to more robustly open the
- header file. We don't have access to the real include paths in this context,
- so we need this guesswork here.
-
- Known bugs: tools/base/bar.cc and base/bar.h belong to the same module
- according to this implementation. Because of this, this function gives
- some false positives. This should be sufficiently rare in practice.
-
- Args:
- filename_cc: is the path for the .cc file
- filename_h: is the path for the header path
-
- Returns:
- Tuple with a bool and a string:
- bool: True if filename_cc and filename_h belong to the same module.
- string: the additional prefix needed to open the header file.
- """
-
- if not filename_cc.endswith('.cc'):
- return (False, '')
- filename_cc = filename_cc[:-len('.cc')]
- if filename_cc.endswith('_unittest'):
- filename_cc = filename_cc[:-len('_unittest')]
- elif filename_cc.endswith('_test'):
- filename_cc = filename_cc[:-len('_test')]
- filename_cc = filename_cc.replace('/public/', '/')
- filename_cc = filename_cc.replace('/internal/', '/')
-
- if not filename_h.endswith('.h'):
- return (False, '')
- filename_h = filename_h[:-len('.h')]
- if filename_h.endswith('-inl'):
- filename_h = filename_h[:-len('-inl')]
- filename_h = filename_h.replace('/public/', '/')
- filename_h = filename_h.replace('/internal/', '/')
-
- files_belong_to_same_module = filename_cc.endswith(filename_h)
- common_path = ''
- if files_belong_to_same_module:
- common_path = filename_cc[:-len(filename_h)]
- return files_belong_to_same_module, common_path
-
-
-def UpdateIncludeState(filename, include_dict, io=codecs):
- """Fill up the include_dict with new includes found from the file.
-
- Args:
- filename: the name of the header to read.
- include_dict: a dictionary in which the headers are inserted.
- io: The io factory to use to read the file. Provided for testability.
-
- Returns:
- True if a header was successfully added. False otherwise.
- """
- headerfile = None
- try:
- headerfile = io.open(filename, 'r', 'utf8', 'replace')
- except IOError:
- return False
- linenum = 0
- for line in headerfile:
- linenum += 1
- clean_line = CleanseComments(line)
- match = _RE_PATTERN_INCLUDE.search(clean_line)
- if match:
- include = match.group(2)
- include_dict.setdefault(include, linenum)
- return True
-
-
-def CheckForIncludeWhatYouUse(filename, clean_lines, include_state, error,
- io=codecs):
- """Reports for missing stl includes.
-
- This function will output warnings to make sure you are including the headers
- necessary for the stl containers and functions that you use. We only give one
- reason to include a header. For example, if you use both equal_to<> and
- less<> in a .h file, only one (the latter in the file) of these will be
- reported as a reason to include the <functional>.
-
- Args:
- filename: The name of the current file.
- clean_lines: A CleansedLines instance containing the file.
- include_state: An _IncludeState instance.
- error: The function to call with any errors found.
- io: The IO factory to use to read the header file. Provided for unittest
- injection.
- """
- required = {} # A map of header name to linenumber and the template entity.
- # Example of required: { '<functional>': (1219, 'less<>') }
-
- for linenum in xrange(clean_lines.NumLines()):
- line = clean_lines.elided[linenum]
- if not line or line[0] == '#':
- continue
-
- # String is special -- it is a non-templatized type in STL.
- matched = _RE_PATTERN_STRING.search(line)
- if matched:
- # Don't warn about strings in non-STL namespaces:
- # (We check only the first match per line; good enough.)
- prefix = line[:matched.start()]
- if prefix.endswith('std::') or not prefix.endswith('::'):
- required['<string>'] = (linenum, 'string')
-
- for pattern, template, header in _re_pattern_algorithm_header:
- if pattern.search(line):
- required[header] = (linenum, template)
-
- # The following function is just a speed up, no semantics are changed.
- if not '<' in line: # Reduces the cpu time usage by skipping lines.
- continue
-
- for pattern, template, header in _re_pattern_templates:
- if pattern.search(line):
- required[header] = (linenum, template)
-
- # The policy is that if you #include something in foo.h you don't need to
- # include it again in foo.cc. Here, we will look at possible includes.
- # Let's flatten the include_state include_list and copy it into a dictionary.
- include_dict = dict([item for sublist in include_state.include_list
- for item in sublist])
-
- # Did we find the header for this file (if any) and successfully load it?
- header_found = False
-
- # Use the absolute path so that matching works properly.
- abs_filename = FileInfo(filename).FullName()
-
- # For Emacs's flymake.
- # If cpplint is invoked from Emacs's flymake, a temporary file is generated
- # by flymake and that file name might end with '_flymake.cc'. In that case,
- # restore original file name here so that the corresponding header file can be
- # found.
- # e.g. If the file name is 'foo_flymake.cc', we should search for 'foo.h'
- # instead of 'foo_flymake.h'
- abs_filename = re.sub(r'_flymake\.cc$', '.cc', abs_filename)
-
- # include_dict is modified during iteration, so we iterate over a copy of
- # the keys.
- header_keys = include_dict.keys()
- for header in header_keys:
- (same_module, common_path) = FilesBelongToSameModule(abs_filename, header)
- fullpath = common_path + header
- if same_module and UpdateIncludeState(fullpath, include_dict, io):
- header_found = True
-
- # If we can't find the header file for a .cc, assume it's because we don't
- # know where to look. In that case we'll give up as we're not sure they
- # didn't include it in the .h file.
- # TODO(unknown): Do a better job of finding .h files so we are confident that
- # not having the .h file means there isn't one.
- if filename.endswith('.cc') and not header_found:
- return
-
- # All the lines have been processed, report the errors found.
- for required_header_unstripped in required:
- template = required[required_header_unstripped][1]
- if required_header_unstripped.strip('<>"') not in include_dict:
- error(filename, required[required_header_unstripped][0],
- 'build/include_what_you_use', 4,
- 'Add #include ' + required_header_unstripped + ' for ' + template)
-
-
-_RE_PATTERN_EXPLICIT_MAKEPAIR = re.compile(r'\bmake_pair\s*<')
-
-
-def CheckMakePairUsesDeduction(filename, clean_lines, linenum, error):
- """Check that make_pair's template arguments are deduced.
-
- G++ 4.6 in C++11 mode fails badly if make_pair's template arguments are
- specified explicitly, and such use isn't intended in any case.
-
- Args:
- filename: The name of the current file.
- clean_lines: A CleansedLines instance containing the file.
- linenum: The number of the line to check.
- error: The function to call with any errors found.
- """
- line = clean_lines.elided[linenum]
- match = _RE_PATTERN_EXPLICIT_MAKEPAIR.search(line)
- if match:
- error(filename, linenum, 'build/explicit_make_pair',
- 4, # 4 = high confidence
- 'For C++11-compatibility, omit template arguments from make_pair'
- ' OR use pair directly OR if appropriate, construct a pair directly')
-
-
-def CheckDefaultLambdaCaptures(filename, clean_lines, linenum, error):
- """Check that default lambda captures are not used.
-
- Args:
- filename: The name of the current file.
- clean_lines: A CleansedLines instance containing the file.
- linenum: The number of the line to check.
- error: The function to call with any errors found.
- """
- line = clean_lines.elided[linenum]
-
- # A lambda introducer specifies a default capture if it starts with "[="
- # or if it starts with "[&" _not_ followed by an identifier.
- match = Match(r'^(.*)\[\s*(?:=|&[^\w])', line)
- if match:
- # Found a potential error, check what comes after the lambda-introducer.
- # If it's not open parenthesis (for lambda-declarator) or open brace
- # (for compound-statement), it's not a lambda.
- line, _, pos = CloseExpression(clean_lines, linenum, len(match.group(1)))
- if pos >= 0 and Match(r'^\s*[{(]', line[pos:]):
- error(filename, linenum, 'build/c++11',
- 4, # 4 = high confidence
- 'Default lambda captures are an unapproved C++ feature.')
-
-
-def CheckRedundantVirtual(filename, clean_lines, linenum, error):
- """Check if line contains a redundant "virtual" function-specifier.
-
- Args:
- filename: The name of the current file.
- clean_lines: A CleansedLines instance containing the file.
- linenum: The number of the line to check.
- error: The function to call with any errors found.
- """
- # Look for "virtual" on current line.
- line = clean_lines.elided[linenum]
- virtual = Match(r'^(.*)(\bvirtual\b)(.*)$', line)
- if not virtual: return
-
- # Ignore "virtual" keywords that are near access-specifiers. These
- # are only used in class base-specifier and do not apply to member
- # functions.
- if (Search(r'\b(public|protected|private)\s+$', virtual.group(1)) or
- Match(r'^\s+(public|protected|private)\b', virtual.group(3))):
- return
-
- # Ignore the "virtual" keyword from virtual base classes. Usually
- # there is a column on the same line in these cases (virtual base
- # classes are rare in google3 because multiple inheritance is rare).
- if Match(r'^.*[^:]:[^:].*$', line): return
-
- # Look for the next opening parenthesis. This is the start of the
- # parameter list (possibly on the next line shortly after virtual).
- # TODO(unknown): doesn't work if there are virtual functions with
- # decltype() or other things that use parentheses, but csearch suggests
- # that this is rare.
- end_col = -1
- end_line = -1
- start_col = len(virtual.group(2))
- for start_line in xrange(linenum, min(linenum + 3, clean_lines.NumLines())):
- line = clean_lines.elided[start_line][start_col:]
- parameter_list = Match(r'^([^(]*)\(', line)
- if parameter_list:
- # Match parentheses to find the end of the parameter list
- (_, end_line, end_col) = CloseExpression(
- clean_lines, start_line, start_col + len(parameter_list.group(1)))
- break
- start_col = 0
-
- if end_col < 0:
- return # Couldn't find end of parameter list, give up
-
- # Look for "override" or "final" after the parameter list
- # (possibly on the next few lines).
- for i in xrange(end_line, min(end_line + 3, clean_lines.NumLines())):
- line = clean_lines.elided[i][end_col:]
- match = Search(r'\b(override|final)\b', line)
- if match:
- error(filename, linenum, 'readability/inheritance', 4,
- ('"virtual" is redundant since function is '
- 'already declared as "%s"' % match.group(1)))
-
- # Set end_col to check whole lines after we are done with the
- # first line.
- end_col = 0
- if Search(r'[^\w]\s*$', line):
- break
-
-
-def CheckRedundantOverrideOrFinal(filename, clean_lines, linenum, error):
- """Check if line contains a redundant "override" or "final" virt-specifier.
-
- Args:
- filename: The name of the current file.
- clean_lines: A CleansedLines instance containing the file.
- linenum: The number of the line to check.
- error: The function to call with any errors found.
- """
- # Look for closing parenthesis nearby. We need one to confirm where
- # the declarator ends and where the virt-specifier starts to avoid
- # false positives.
- line = clean_lines.elided[linenum]
- declarator_end = line.rfind(')')
- if declarator_end >= 0:
- fragment = line[declarator_end:]
- else:
- if linenum > 1 and clean_lines.elided[linenum - 1].rfind(')') >= 0:
- fragment = line
- else:
- return
-
- # Check that at most one of "override" or "final" is present, not both
- if Search(r'\boverride\b', fragment) and Search(r'\bfinal\b', fragment):
- error(filename, linenum, 'readability/inheritance', 4,
- ('"override" is redundant since function is '
- 'already declared as "final"'))
-
-
-
-
-# Returns true if we are at a new block, and it is directly
-# inside of a namespace.
-def IsBlockInNameSpace(nesting_state, is_forward_declaration):
- """Checks that the new block is directly in a namespace.
-
- Args:
- nesting_state: The _NestingState object that contains info about our state.
- is_forward_declaration: If the class is a forward declared class.
- Returns:
- Whether or not the new block is directly in a namespace.
- """
- if is_forward_declaration:
- if len(nesting_state.stack) >= 1 and (
- isinstance(nesting_state.stack[-1], _NamespaceInfo)):
- return True
- else:
- return False
-
- return (len(nesting_state.stack) > 1 and
- nesting_state.stack[-1].check_namespace_indentation and
- isinstance(nesting_state.stack[-2], _NamespaceInfo))
-
-
-def ShouldCheckNamespaceIndentation(nesting_state, is_namespace_indent_item,
- raw_lines_no_comments, linenum):
- """This method determines if we should apply our namespace indentation check.
-
- Args:
- nesting_state: The current nesting state.
- is_namespace_indent_item: If we just put a new class on the stack, True.
- If the top of the stack is not a class, or we did not recently
- add the class, False.
- raw_lines_no_comments: The lines without the comments.
- linenum: The current line number we are processing.
-
- Returns:
- True if we should apply our namespace indentation check. Currently, it
- only works for classes and namespaces inside of a namespace.
- """
-
- is_forward_declaration = IsForwardClassDeclaration(raw_lines_no_comments,
- linenum)
-
- if not (is_namespace_indent_item or is_forward_declaration):
- return False
-
- # If we are in a macro, we do not want to check the namespace indentation.
- if IsMacroDefinition(raw_lines_no_comments, linenum):
- return False
-
- return IsBlockInNameSpace(nesting_state, is_forward_declaration)
-
-
-# Call this method if the line is directly inside of a namespace.
-# If the line above is blank (excluding comments) or the start of
-# an inner namespace, it cannot be indented.
-def CheckItemIndentationInNamespace(filename, raw_lines_no_comments, linenum,
- error):
- line = raw_lines_no_comments[linenum]
- if Match(r'^\s+', line):
- error(filename, linenum, 'runtime/indentation_namespace', 4,
- 'Do not indent within a namespace')
-
-
-def ProcessLine(filename, file_extension, clean_lines, line,
- include_state, function_state, nesting_state, error,
- extra_check_functions=[]):
- """Processes a single line in the file.
-
- Args:
- filename: Filename of the file that is being processed.
- file_extension: The extension (dot not included) of the file.
- clean_lines: An array of strings, each representing a line of the file,
- with comments stripped.
- line: Number of line being processed.
- include_state: An _IncludeState instance in which the headers are inserted.
- function_state: A _FunctionState instance which counts function lines, etc.
- nesting_state: A NestingState instance which maintains information about
- the current stack of nested blocks being parsed.
- error: A callable to which errors are reported, which takes 4 arguments:
- filename, line number, error level, and message
- extra_check_functions: An array of additional check functions that will be
- run on each source line. Each function takes 4
- arguments: filename, clean_lines, line, error
- """
- raw_lines = clean_lines.raw_lines
- ParseNolintSuppressions(filename, raw_lines[line], line, error)
- nesting_state.Update(filename, clean_lines, line, error)
- CheckForNamespaceIndentation(filename, nesting_state, clean_lines, line,
- error)
- if nesting_state.InAsmBlock(): return
- CheckForFunctionLengths(filename, clean_lines, line, function_state, error)
- CheckForMultilineCommentsAndStrings(filename, clean_lines, line, error)
- CheckStyle(filename, clean_lines, line, file_extension, nesting_state, error)
- CheckLanguage(filename, clean_lines, line, file_extension, include_state,
- nesting_state, error)
- CheckForNonConstReference(filename, clean_lines, line, nesting_state, error)
- CheckForNonStandardConstructs(filename, clean_lines, line,
- nesting_state, error)
- CheckVlogArguments(filename, clean_lines, line, error)
- CheckPosixThreading(filename, clean_lines, line, error)
- CheckInvalidIncrement(filename, clean_lines, line, error)
- CheckMakePairUsesDeduction(filename, clean_lines, line, error)
- CheckDefaultLambdaCaptures(filename, clean_lines, line, error)
- CheckRedundantVirtual(filename, clean_lines, line, error)
- CheckRedundantOverrideOrFinal(filename, clean_lines, line, error)
- for check_fn in extra_check_functions:
- check_fn(filename, clean_lines, line, error)
-
-def FlagCxx11Features(filename, clean_lines, linenum, error):
- """Flag those c++11 features that we only allow in certain places.
-
- Args:
- filename: The name of the current file.
- clean_lines: A CleansedLines instance containing the file.
- linenum: The number of the line to check.
- error: The function to call with any errors found.
- """
- line = clean_lines.elided[linenum]
-
- # Flag unapproved C++11 headers.
- include = Match(r'\s*#\s*include\s+[<"]([^<"]+)[">]', line)
- if include and include.group(1) in ('cfenv',
- 'condition_variable',
- 'fenv.h',
- 'future',
- 'mutex',
- 'thread',
- 'chrono',
- 'ratio',
- 'regex',
- 'system_error',
- ):
- error(filename, linenum, 'build/c++11', 5,
- ('<%s> is an unapproved C++11 header.') % include.group(1))
-
- # The only place where we need to worry about C++11 keywords and library
- # features in preprocessor directives is in macro definitions.
- if Match(r'\s*#', line) and not Match(r'\s*#\s*define\b', line): return
-
- # These are classes and free functions. The classes are always
- # mentioned as std::*, but we only catch the free functions if
- # they're not found by ADL. They're alphabetical by header.
- for top_name in (
- # type_traits
- 'alignment_of',
- 'aligned_union',
- ):
- if Search(r'\bstd::%s\b' % top_name, line):
- error(filename, linenum, 'build/c++11', 5,
- ('std::%s is an unapproved C++11 class or function. Send c-style '
- 'an example of where it would make your code more readable, and '
- 'they may let you use it.') % top_name)
-
-
-def ProcessFileData(filename, file_extension, lines, error,
- extra_check_functions=[]):
- """Performs lint checks and reports any errors to the given error function.
-
- Args:
- filename: Filename of the file that is being processed.
- file_extension: The extension (dot not included) of the file.
- lines: An array of strings, each representing a line of the file, with the
- last element being empty if the file is terminated with a newline.
- error: A callable to which errors are reported, which takes 4 arguments:
- filename, line number, error level, and message
- extra_check_functions: An array of additional check functions that will be
- run on each source line. Each function takes 4
- arguments: filename, clean_lines, line, error
- """
- lines = (['// marker so line numbers and indices both start at 1'] + lines +
- ['// marker so line numbers end in a known way'])
-
- include_state = _IncludeState()
- function_state = _FunctionState()
- nesting_state = NestingState()
-
- ResetNolintSuppressions()
-
- CheckForCopyright(filename, lines, error)
-
- RemoveMultiLineComments(filename, lines, error)
- clean_lines = CleansedLines(lines)
-
- if file_extension == 'h':
- CheckForHeaderGuard(filename, clean_lines, error)
-
- for line in xrange(clean_lines.NumLines()):
- ProcessLine(filename, file_extension, clean_lines, line,
- include_state, function_state, nesting_state, error,
- extra_check_functions)
- FlagCxx11Features(filename, clean_lines, line, error)
- nesting_state.CheckCompletedBlocks(filename, error)
-
- CheckForIncludeWhatYouUse(filename, clean_lines, include_state, error)
-
- # Check that the .cc file has included its header if it exists.
- if file_extension == 'cc':
- CheckHeaderFileIncluded(filename, include_state, error)
-
- # We check here rather than inside ProcessLine so that we see raw
- # lines rather than "cleaned" lines.
- CheckForBadCharacters(filename, lines, error)
-
- CheckForNewlineAtEOF(filename, lines, error)
-
-def ProcessConfigOverrides(filename):
- """ Loads the configuration files and processes the config overrides.
-
- Args:
- filename: The name of the file being processed by the linter.
-
- Returns:
- False if the current |filename| should not be processed further.
- """
-
- abs_filename = os.path.abspath(filename)
- cfg_filters = []
- keep_looking = True
- while keep_looking:
- abs_path, base_name = os.path.split(abs_filename)
- if not base_name:
- break # Reached the root directory.
-
- cfg_file = os.path.join(abs_path, "CPPLINT.cfg")
- abs_filename = abs_path
- if not os.path.isfile(cfg_file):
- continue
-
- try:
- with open(cfg_file) as file_handle:
- for line in file_handle:
- line, _, _ = line.partition('#') # Remove comments.
- if not line.strip():
- continue
-
- name, _, val = line.partition('=')
- name = name.strip()
- val = val.strip()
- if name == 'set noparent':
- keep_looking = False
- elif name == 'filter':
- cfg_filters.append(val)
- elif name == 'exclude_files':
- # When matching exclude_files pattern, use the base_name of
- # the current file name or the directory name we are processing.
- # For example, if we are checking for lint errors in /foo/bar/baz.cc
- # and we found the .cfg file at /foo/CPPLINT.cfg, then the config
- # file's "exclude_files" filter is meant to be checked against "bar"
- # and not "baz" nor "bar/baz.cc".
- if base_name:
- pattern = re.compile(val)
- if pattern.match(base_name):
- sys.stderr.write('Ignoring "%s": file excluded by "%s". '
- 'File path component "%s" matches '
- 'pattern "%s"\n' %
- (filename, cfg_file, base_name, val))
- return False
- elif name == 'linelength':
- global _line_length
- try:
- _line_length = int(val)
- except ValueError:
- sys.stderr.write('Line length must be numeric.')
- else:
- sys.stderr.write(
- 'Invalid configuration option (%s) in file %s\n' %
- (name, cfg_file))
-
- except IOError:
- sys.stderr.write(
- "Skipping config file '%s': Can't open for reading\n" % cfg_file)
- keep_looking = False
-
- # Apply all the accumulated filters in reverse order (top-level directory
- # config options having the least priority).
- for filter in reversed(cfg_filters):
- _AddFilters(filter)
-
- return True
-
-
-def ProcessFile(filename, vlevel, extra_check_functions=[]):
- """Does google-lint on a single file.
-
- Args:
- filename: The name of the file to parse.
-
- vlevel: The level of errors to report. Every error of confidence
- >= verbose_level will be reported. 0 is a good default.
-
- extra_check_functions: An array of additional check functions that will be
- run on each source line. Each function takes 4
- arguments: filename, clean_lines, line, error
- """
-
- _SetVerboseLevel(vlevel)
- _BackupFilters()
-
- if not ProcessConfigOverrides(filename):
- _RestoreFilters()
- return
-
- lf_lines = []
- crlf_lines = []
- try:
- # Support the UNIX convention of using "-" for stdin. Note that
- # we are not opening the file with universal newline support
- # (which codecs doesn't support anyway), so the resulting lines do
- # contain trailing '\r' characters if we are reading a file that
- # has CRLF endings.
- # If after the split a trailing '\r' is present, it is removed
- # below.
- if filename == '-':
- lines = codecs.StreamReaderWriter(sys.stdin,
- codecs.getreader('utf8'),
- codecs.getwriter('utf8'),
- 'replace').read().split('\n')
- else:
- lines = codecs.open(filename, 'r', 'utf8', 'replace').read().split('\n')
-
- # Remove trailing '\r'.
- # The -1 accounts for the extra trailing blank line we get from split()
- for linenum in range(len(lines) - 1):
- if lines[linenum].endswith('\r'):
- lines[linenum] = lines[linenum].rstrip('\r')
- crlf_lines.append(linenum + 1)
- else:
- lf_lines.append(linenum + 1)
-
- except IOError:
- sys.stderr.write(
- "Skipping input '%s': Can't open for reading\n" % filename)
- _RestoreFilters()
- return
-
- # Note, if no dot is found, this will give the entire filename as the ext.
- file_extension = filename[filename.rfind('.') + 1:]
-
- # When reading from stdin, the extension is unknown, so no cpplint tests
- # should rely on the extension.
- if filename != '-' and file_extension not in _valid_extensions:
- sys.stderr.write('Ignoring %s; not a valid file name '
- '(%s)\n' % (filename, ', '.join(_valid_extensions)))
- else:
- ProcessFileData(filename, file_extension, lines, Error,
- extra_check_functions)
-
- # If end-of-line sequences are a mix of LF and CR-LF, issue
- # warnings on the lines with CR.
- #
- # Don't issue any warnings if all lines are uniformly LF or CR-LF,
- # since critique can handle these just fine, and the style guide
- # doesn't dictate a particular end of line sequence.
- #
- # We can't depend on os.linesep to determine what the desired
- # end-of-line sequence should be, since that will return the
- # server-side end-of-line sequence.
- if lf_lines and crlf_lines:
- # Warn on every line with CR. An alternative approach might be to
- # check whether the file is mostly CRLF or just LF, and warn on the
- # minority, we bias toward LF here since most tools prefer LF.
- for linenum in crlf_lines:
- Error(filename, linenum, 'whitespace/newline', 1,
- 'Unexpected \\r (^M) found; better to use only \\n')
-
- sys.stderr.write('Done processing %s\n' % filename)
- _RestoreFilters()
-
-
-def PrintUsage(message):
- """Prints a brief usage string and exits, optionally with an error message.
-
- Args:
- message: The optional error message.
- """
- sys.stderr.write(_USAGE)
- if message:
- sys.exit('\nFATAL ERROR: ' + message)
- else:
- sys.exit(1)
-
-
-def PrintCategories():
- """Prints a list of all the error-categories used by error messages.
-
- These are the categories used to filter messages via --filter.
- """
- sys.stderr.write(''.join(' %s\n' % cat for cat in _ERROR_CATEGORIES))
- sys.exit(0)
-
-
-def ParseArguments(args):
- """Parses the command line arguments.
-
- This may set the output format and verbosity level as side-effects.
-
- Args:
- args: The command line arguments:
-
- Returns:
- The list of filenames to lint.
- """
- try:
- (opts, filenames) = getopt.getopt(args, '', ['help', 'output=', 'verbose=',
- 'counting=',
- 'filter=',
- 'root=',
- 'linelength=',
- 'extensions='])
- except getopt.GetoptError:
- PrintUsage('Invalid arguments.')
-
- verbosity = _VerboseLevel()
- output_format = _OutputFormat()
- filters = ''
- counting_style = ''
-
- for (opt, val) in opts:
- if opt == '--help':
- PrintUsage(None)
- elif opt == '--output':
- if val not in ('emacs', 'vs7', 'eclipse'):
- PrintUsage('The only allowed output formats are emacs, vs7 and eclipse.')
- output_format = val
- elif opt == '--verbose':
- verbosity = int(val)
- elif opt == '--filter':
- filters = val
- if not filters:
- PrintCategories()
- elif opt == '--counting':
- if val not in ('total', 'toplevel', 'detailed'):
- PrintUsage('Valid counting options are total, toplevel, and detailed')
- counting_style = val
- elif opt == '--root':
- global _root
- _root = val
- elif opt == '--linelength':
- global _line_length
- try:
- _line_length = int(val)
- except ValueError:
- PrintUsage('Line length must be digits.')
- elif opt == '--extensions':
- global _valid_extensions
- try:
- _valid_extensions = set(val.split(','))
- except ValueError:
- PrintUsage('Extensions must be comma seperated list.')
-
- if not filenames:
- PrintUsage('No files were specified.')
-
- _SetOutputFormat(output_format)
- _SetVerboseLevel(verbosity)
- _SetFilters(filters)
- _SetCountingStyle(counting_style)
-
- return filenames
-
-
-def main():
- filenames = ParseArguments(sys.argv[1:])
-
- # Change stderr to write with replacement characters so we don't die
- # if we try to print something containing non-ASCII characters.
- sys.stderr = codecs.StreamReaderWriter(sys.stderr,
- codecs.getreader('utf8'),
- codecs.getwriter('utf8'),
- 'replace')
-
- _cpplint_state.ResetErrorCounts()
- for filename in filenames:
- ProcessFile(filename, _cpplint_state.verbose_level)
- _cpplint_state.PrintErrorCounts()
-
- sys.exit(_cpplint_state.error_count > 0)
-
-
-if __name__ == '__main__':
- main()
diff --git a/build-support/run-clang-format.sh b/build-support/run-clang-format.sh
deleted file mode 100755
index 01ddab2..0000000
--- a/build-support/run-clang-format.sh
+++ /dev/null
@@ -1,42 +0,0 @@
-#!/bin/bash
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-# Runs clang format in the given directory
-# Arguments:
-# $1 - Path to the source tree
-# $2 - Path to the clang format binary
-# $3 - Apply fixes (will raise an error if false and not there where changes)
-# $ARGN - Files to run clang format on
-#
-SOURCE_DIR=$1
-shift
-CLANG_FORMAT=$1
-shift
-APPLY_FIXES=$1
-shift
-
-# clang format will only find its configuration if we are in
-# the source tree or in a path relative to the source tree
-pushd $SOURCE_DIR
-if [ "$APPLY_FIXES" == "1" ]; then
- $CLANG_FORMAT -i $@
-else
-
- NUM_CORRECTIONS=`$CLANG_FORMAT -output-replacements-xml $@ | grep offset | wc -l`
- if [ "$NUM_CORRECTIONS" -gt "0" ]; then
- echo "clang-format suggested changes, please run 'make format'!!!!"
- exit 1
- fi
-fi
-popd
diff --git a/build-support/run-clang-tidy.sh b/build-support/run-clang-tidy.sh
deleted file mode 100755
index 2a4b1c0..0000000
--- a/build-support/run-clang-tidy.sh
+++ /dev/null
@@ -1,40 +0,0 @@
-#!/bin/bash
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-#
-# Runs clang format in the given directory
-# Arguments:
-# $1 - Path to the clang tidy binary
-# $2 - Path to the compile_commands.json to use
-# $3 - Apply fixes (will raise an error if false and not there where changes)
-# $ARGN - Files to run clang-tidy on
-#
-CLANG_TIDY=$1
-shift
-COMPILE_COMMANDS=$1
-shift
-APPLY_FIXES=$1
-shift
-
-# clang format will only find its configuration if we are in
-# the source tree or in a path relative to the source tree
-if [ "$APPLY_FIXES" == "1" ]; then
- $CLANG_TIDY -p $COMPILE_COMMANDS -fix $@
-else
- NUM_CORRECTIONS=`$CLANG_TIDY -p $COMPILE_COMMANDS $@ 2>&1 | grep -v Skipping | grep "warnings* generated" | wc -l`
- if [ "$NUM_CORRECTIONS" -gt "0" ]; then
- echo "clang-tidy had suggested fixes. Please fix these!!!"
- exit 1
- fi
-fi
diff --git a/build-support/run-test.sh b/build-support/run-test.sh
deleted file mode 100755
index 7c3d8f6..0000000
--- a/build-support/run-test.sh
+++ /dev/null
@@ -1,229 +0,0 @@
-#!/bin/bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-# Script which wraps running a test and redirects its output to a
-# test log directory.
-#
-# Arguments:
-# $1 - Base path for logs/artifacts.
-# $2 - type of test (e.g. test or benchmark)
-# $3 - path to executable
-# $ARGN - arguments for executable
-#
-
-OUTPUT_ROOT=$1
-shift
-ROOT=$(cd $(dirname $BASH_SOURCE)/..; pwd)
-
-TEST_LOGDIR=$OUTPUT_ROOT/build/$1-logs
-mkdir -p $TEST_LOGDIR
-
-RUN_TYPE=$1
-shift
-TEST_DEBUGDIR=$OUTPUT_ROOT/build/$RUN_TYPE-debug
-mkdir -p $TEST_DEBUGDIR
-
-TEST_DIRNAME=$(cd $(dirname $1); pwd)
-TEST_FILENAME=$(basename $1)
-shift
-TEST_EXECUTABLE="$TEST_DIRNAME/$TEST_FILENAME"
-TEST_NAME=$(echo $TEST_FILENAME | perl -pe 's/\..+?$//') # Remove path and extension (if any).
-
-# We run each test in its own subdir to avoid core file related races.
-TEST_WORKDIR=$OUTPUT_ROOT/build/test-work/$TEST_NAME
-mkdir -p $TEST_WORKDIR
-pushd $TEST_WORKDIR >/dev/null || exit 1
-rm -f *
-
-set -o pipefail
-
-LOGFILE=$TEST_LOGDIR/$TEST_NAME.txt
-XMLFILE=$TEST_LOGDIR/$TEST_NAME.xml
-
-TEST_EXECUTION_ATTEMPTS=1
-
-# Remove both the uncompressed output, so the developer doesn't accidentally get confused
-# and read output from a prior test run.
-rm -f $LOGFILE $LOGFILE.gz
-
-pipe_cmd=cat
-
-# Allow for collecting core dumps.
-PARQUET_TEST_ULIMIT_CORE=${PARQUET_TEST_ULIMIT_CORE:-0}
-ulimit -c $PARQUET_TEST_ULIMIT_CORE
-
-
-function setup_sanitizers() {
- # Sets environment variables for different sanitizers (it configures how) the run_tests. Function works.
-
- # Configure TSAN (ignored if this isn't a TSAN build).
- #
- # Deadlock detection (new in clang 3.5) is disabled because:
- # 1. The clang 3.5 deadlock detector crashes in some unit tests. It
- # needs compiler-rt commits c4c3dfd, 9a8efe3, and possibly others.
- # 2. Many unit tests report lock-order-inversion warnings; they should be
- # fixed before reenabling the detector.
- TSAN_OPTIONS="$TSAN_OPTIONS detect_deadlocks=0"
- TSAN_OPTIONS="$TSAN_OPTIONS suppressions=$ROOT/build-support/tsan-suppressions.txt"
- TSAN_OPTIONS="$TSAN_OPTIONS history_size=7"
- export TSAN_OPTIONS
-
- # Enable leak detection even under LLVM 3.4, where it was disabled by default.
- # This flag only takes effect when running an ASAN build.
- ASAN_OPTIONS="$ASAN_OPTIONS detect_leaks=1"
- export ASAN_OPTIONS
-
- # Set up suppressions for LeakSanitizer
- LSAN_OPTIONS="$LSAN_OPTIONS suppressions=$ROOT/build-support/lsan-suppressions.txt"
- export LSAN_OPTIONS
-
- # Suppressions require symbolization. We'll default to using the symbolizer in
- # thirdparty.
- if [ -z "$ASAN_SYMBOLIZER_PATH" ]; then
- export ASAN_SYMBOLIZER_PATH=$(find $NATIVE_TOOLCHAIN/llvm-3.7.0/bin -name llvm-symbolizer)
- fi
-}
-
-function run_test() {
- # Run gtest style tests with sanitizers if they are setup appropriately.
-
- # gtest won't overwrite old junit test files, resulting in a build failure
- # even when retries are successful.
- rm -f $XMLFILE
-
- # This is needed for test execution on macOS with shared arrow linkage build
- # via external project.
- export LD_LIBRARY_PATH=${TEST_DIRNAME}:$LD_LIBRARY_PATH
- $TEST_EXECUTABLE "$@" 2>&1 \
- | c++filt \
- | $ROOT/build-support/stacktrace_addr2line.pl $TEST_EXECUTABLE \
- | $pipe_cmd 2>&1 | tee $LOGFILE
- STATUS=$?
-
- # TSAN doesn't always exit with a non-zero exit code due to a bug:
- # mutex errors don't get reported through the normal error reporting infrastructure.
- # So we make sure to detect this and exit 1.
- #
- # Additionally, certain types of failures won't show up in the standard JUnit
- # XML output from gtest. We assume that gtest knows better than us and our
- # regexes in most cases, but for certain errors we delete the resulting xml
- # file and let our own post-processing step regenerate it.
- export GREP=$(which egrep)
- if zgrep --silent "ThreadSanitizer|Leak check.*detected leaks" $LOGFILE ; then
- echo ThreadSanitizer or leak check failures in $LOGFILE
- STATUS=1
- rm -f $XMLFILE
- fi
-}
-
-function post_process_tests() {
- # If we have a LeakSanitizer report, and XML reporting is configured, add a new test
- # case result to the XML file for the leak report. Otherwise Jenkins won't show
- # us which tests had LSAN errors.
- if zgrep --silent "ERROR: LeakSanitizer: detected memory leaks" $LOGFILE ; then
- echo Test had memory leaks. Editing XML
- perl -p -i -e '
- if (m#</testsuite>#) {
- print "<testcase name=\"LeakSanitizer\" status=\"run\" classname=\"LSAN\">\n";
- print " <failure message=\"LeakSanitizer failed\" type=\"\">\n";
- print " See txt log file for details\n";
- print " </failure>\n";
- print "</testcase>\n";
- }' $XMLFILE
- fi
-}
-
-function run_other() {
- # Generic run function for test like executables that aren't actually gtest
- $TEST_EXECUTABLE "$@" 2>&1 | $pipe_cmd > $LOGFILE
- STATUS=$?
-}
-
-if [ $RUN_TYPE = "test" ]; then
- setup_sanitizers
-fi
-
-# Run the actual test.
-for ATTEMPT_NUMBER in $(seq 1 $TEST_EXECUTION_ATTEMPTS) ; do
- if [ $ATTEMPT_NUMBER -lt $TEST_EXECUTION_ATTEMPTS ]; then
- # If the test fails, the test output may or may not be left behind,
- # depending on whether the test cleaned up or exited immediately. Either
- # way we need to clean it up. We do this by comparing the data directory
- # contents before and after the test runs, and deleting anything new.
- #
- # The comm program requires that its two inputs be sorted.
- TEST_TMPDIR_BEFORE=$(find $TEST_TMPDIR -maxdepth 1 -type d | sort)
- fi
-
- if [ $ATTEMPT_NUMBER -lt $TEST_EXECUTION_ATTEMPTS ]; then
- # Now delete any new test output.
- TEST_TMPDIR_AFTER=$(find $TEST_TMPDIR -maxdepth 1 -type d | sort)
- DIFF=$(comm -13 <(echo "$TEST_TMPDIR_BEFORE") \
- <(echo "$TEST_TMPDIR_AFTER"))
- for DIR in $DIFF; do
- # Multiple tests may be running concurrently. To avoid deleting the
- # wrong directories, constrain to only directories beginning with the
- # test name.
- #
- # This may delete old test directories belonging to this test, but
- # that's not typically a concern when rerunning flaky tests.
- if [[ $DIR =~ ^$TEST_TMPDIR/$TEST_NAME ]]; then
- echo Deleting leftover flaky test directory "$DIR"
- rm -Rf "$DIR"
- fi
- done
- fi
- echo "Running $TEST_NAME, redirecting output into $LOGFILE" \
- "(attempt ${ATTEMPT_NUMBER}/$TEST_EXECUTION_ATTEMPTS)"
- if [ $RUN_TYPE = "test" ]; then
- run_test $*
- else
- run_other $*
- fi
- if [ "$STATUS" -eq "0" ]; then
- break
- elif [ "$ATTEMPT_NUMBER" -lt "$TEST_EXECUTION_ATTEMPTS" ]; then
- echo Test failed attempt number $ATTEMPT_NUMBER
- echo Will retry...
- fi
-done
-
-if [ $RUN_TYPE = "test" ]; then
- post_process_tests
-fi
-
-# Capture and compress core file and binary.
-COREFILES=$(ls | grep ^core)
-if [ -n "$COREFILES" ]; then
- echo Found core dump. Saving executable and core files.
- gzip < $TEST_EXECUTABLE > "$TEST_DEBUGDIR/$TEST_NAME.gz" || exit $?
- for COREFILE in $COREFILES; do
- gzip < $COREFILE > "$TEST_DEBUGDIR/$TEST_NAME.$COREFILE.gz" || exit $?
- done
- # Pull in any .so files as well.
- for LIB in $(ldd $TEST_EXECUTABLE | grep $ROOT | awk '{print $3}'); do
- LIB_NAME=$(basename $LIB)
- gzip < $LIB > "$TEST_DEBUGDIR/$LIB_NAME.gz" || exit $?
- done
-fi
-
-popd
-rm -Rf $TEST_WORKDIR
-
-exit $STATUS
diff --git a/build-support/run_clang_format.py b/build-support/run_clang_format.py
deleted file mode 100755
index 6dec34b..0000000
--- a/build-support/run_clang_format.py
+++ /dev/null
@@ -1,78 +0,0 @@
-#!/usr/bin/env python
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-import fnmatch
-import os
-import subprocess
-import sys
-
-if len(sys.argv) < 4:
- sys.stderr.write("Usage: %s $CLANG_FORMAT_VERSION exclude_globs.txt "
- "$source_dir\n" %
- sys.argv[0])
- sys.exit(1)
-
-CLANG_FORMAT = sys.argv[1]
-EXCLUDE_GLOBS_FILENAME = sys.argv[2]
-SOURCE_DIR = sys.argv[3]
-
-if len(sys.argv) > 4:
- CHECK_FORMAT = int(sys.argv[4]) == 1
-else:
- CHECK_FORMAT = False
-
-
-exclude_globs = [line.strip() for line in open(EXCLUDE_GLOBS_FILENAME, "r")]
-
-files_to_format = []
-matches = []
-for directory, subdirs, files in os.walk(SOURCE_DIR):
- for name in files:
- name = os.path.join(directory, name)
- if not (name.endswith('.h') or name.endswith('.cc')):
- continue
-
- excluded = False
- for g in exclude_globs:
- if fnmatch.fnmatch(name, g):
- excluded = True
- break
- if not excluded:
- files_to_format.append(name)
-
-if CHECK_FORMAT:
- output = subprocess.check_output([CLANG_FORMAT, '-output-replacements-xml']
- + files_to_format,
- stderr=subprocess.STDOUT).decode('utf8')
-
- to_fix = []
- for line in output.split('\n'):
- if 'offset' in line:
- to_fix.append(line)
-
- if len(to_fix) > 0:
- print("clang-format checks failed, run 'make format' to fix")
- sys.exit(-1)
-else:
- try:
- cmd = [CLANG_FORMAT, '-i'] + files_to_format
- subprocess.check_output(cmd, stderr=subprocess.STDOUT)
- except Exception as e:
- print(e)
- print(' '.join(cmd))
- raise
diff --git a/build-support/stacktrace_addr2line.pl b/build-support/stacktrace_addr2line.pl
deleted file mode 100755
index f407247..0000000
--- a/build-support/stacktrace_addr2line.pl
+++ /dev/null
@@ -1,95 +0,0 @@
-#!/usr/bin/env perl
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#######################################################################
-# This script will convert a stack trace with addresses:
-# @ 0x5fb015 kudu::master::Master::Init()
-# @ 0x5c2d38 kudu::master::MiniMaster::StartOnPorts()
-# @ 0x5c31fa kudu::master::MiniMaster::Start()
-# @ 0x58270a kudu::MiniCluster::Start()
-# @ 0x57dc71 kudu::CreateTableStressTest::SetUp()
-# To one with line numbers:
-# @ 0x5fb015 kudu::master::Master::Init() at /home/mpercy/src/kudu/src/master/master.cc:54
-# @ 0x5c2d38 kudu::master::MiniMaster::StartOnPorts() at /home/mpercy/src/kudu/src/master/mini_master.cc:52
-# @ 0x5c31fa kudu::master::MiniMaster::Start() at /home/mpercy/src/kudu/src/master/mini_master.cc:33
-# @ 0x58270a kudu::MiniCluster::Start() at /home/mpercy/src/kudu/src/integration-tests/mini_cluster.cc:48
-# @ 0x57dc71 kudu::CreateTableStressTest::SetUp() at /home/mpercy/src/kudu/src/integration-tests/create-table-stress-test.cc:61
-#
-# If the script detects that the output is not symbolized, it will also attempt
-# to determine the function names, i.e. it will convert:
-# @ 0x5fb015
-# @ 0x5c2d38
-# @ 0x5c31fa
-# To:
-# @ 0x5fb015 kudu::master::Master::Init() at /home/mpercy/src/kudu/src/master/master.cc:54
-# @ 0x5c2d38 kudu::master::MiniMaster::StartOnPorts() at /home/mpercy/src/kudu/src/master/mini_master.cc:52
-# @ 0x5c31fa kudu::master::MiniMaster::Start() at /home/mpercy/src/kudu/src/master/mini_master.cc:33
-#######################################################################
-use strict;
-use warnings;
-
-if (!@ARGV) {
- die <<EOF
-Usage: $0 executable [stack-trace-file]
-
-This script will read addresses from a file containing stack traces and
-will convert the addresses that conform to the pattern " @ 0x123456" to line
-numbers by calling addr2line on the provided executable.
-If no stack-trace-file is specified, it will take input from stdin.
-EOF
-}
-
-# el6 and other older systems don't support the -p flag,
-# so we do our own "pretty" parsing.
-sub parse_addr2line_output($$) {
- defined(my $output = shift) or die;
- defined(my $lookup_func_name = shift) or die;
- my @lines = grep { $_ ne '' } split("\n", $output);
- my $pretty_str = '';
- if ($lookup_func_name) {
- $pretty_str .= ' ' . $lines[0];
- }
- $pretty_str .= ' at ' . $lines[1];
- return $pretty_str;
-}
-
-my $binary = shift @ARGV;
-if (! -x $binary || ! -r $binary) {
- die "Error: Cannot access executable ($binary)";
-}
-
-# Cache lookups to speed processing of files with repeated trace addresses.
-my %addr2line_map = ();
-
-# Disable stdout buffering
-$| = 1;
-
-# Reading from <ARGV> is magical in Perl.
-while (defined(my $input = <ARGV>)) {
- if ($input =~ /^\s+\@\s+(0x[[:xdigit:]]{6,})(?:\s+(\S+))?/) {
- my $addr = $1;
- my $lookup_func_name = (!defined $2);
- if (!exists($addr2line_map{$addr})) {
- $addr2line_map{$addr} = `addr2line -ifC -e $binary $addr`;
- }
- chomp $input;
- $input .= parse_addr2line_output($addr2line_map{$addr}, $lookup_func_name) . "\n";
- }
- print $input;
-}
-
-exit 0;
diff --git a/ci/before_script_travis.sh b/ci/before_script_travis.sh
deleted file mode 100755
index ce0234c..0000000
--- a/ci/before_script_travis.sh
+++ /dev/null
@@ -1,47 +0,0 @@
-#!/usr/bin/env bash
-
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License. See accompanying LICENSE file.
-
-if [ $TRAVIS_OS_NAME == "osx" ]; then
- brew update > /dev/null
- brew install boost
- brew install openssl
- brew install bison
- export OPENSSL_ROOT_DIR=/usr/local/opt/openssl
- export LD_LIBRARY_PATH=/usr/local/opt/openssl/lib:$LD_LIBRARY_PATH
- export PATH="/usr/local/opt/bison/bin:$PATH"
-else
- # Use a C++11 compiler on Linux
- export CC="gcc-4.9"
- export CXX="g++-4.9"
-fi
-
-export PARQUET_TEST_DATA=$TRAVIS_BUILD_DIR/data
-
-CMAKE_COMMON_FLAGS="-DPARQUET_BUILD_WARNING_LEVEL=CHECKIN"
-
-if [ $PARQUET_TRAVIS_VALGRIND == "1" ]; then
- CMAKE_COMMON_FLAGS="$CMAKE_COMMON_FLAGS -DPARQUET_TEST_MEMCHECK=ON"
-fi
-
-if [ $TRAVIS_OS_NAME == "linux" ]; then
- cmake $CMAKE_COMMON_FLAGS \
- -DPARQUET_CXXFLAGS="$PARQUET_CXXFLAGS" \
- -DPARQUET_BUILD_BENCHMARKS=ON \
- -DPARQUET_GENERATE_COVERAGE=1 \
- $TRAVIS_BUILD_DIR
-else
- cmake $CMAKE_COMMON_FLAGS \
- -DPARQUET_CXXFLAGS="$PARQUET_CXXFLAGS" \
- $TRAVIS_BUILD_DIR
-fi
diff --git a/ci/msvc-build.bat b/ci/msvc-build.bat
deleted file mode 100644
index 7a50c85..0000000
--- a/ci/msvc-build.bat
+++ /dev/null
@@ -1,77 +0,0 @@
-@rem Licensed to the Apache Software Foundation (ASF) under one
-@rem or more contributor license agreements. See the NOTICE file
-@rem distributed with this work for additional information
-@rem regarding copyright ownership. The ASF licenses this file
-@rem to you under the Apache License, Version 2.0 (the
-@rem "License"); you may not use this file except in compliance
-@rem with the License. You may obtain a copy of the License at
-@rem
-@rem http://www.apache.org/licenses/LICENSE-2.0
-@rem
-@rem Unless required by applicable law or agreed to in writing,
-@rem software distributed under the License is distributed on an
-@rem "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-@rem KIND, either express or implied. See the License for the
-@rem specific language governing permissions and limitations
-@rem under the License.
-
-@echo on
-
-mkdir build
-cd build
-
-SET PARQUET_TEST_DATA=%APPVEYOR_BUILD_FOLDER%\data
-set PARQUET_CXXFLAGS=/MP
-
-set PARQUET_USE_STATIC_CRT_OPTION=OFF
-if "%USE_STATIC_CRT%" == "ON" (
- set PARQUET_USE_STATIC_CRT_OPTION=ON
-)
-
-if NOT "%CONFIGURATION%" == "Debug" (
- set PARQUET_CXXFLAGS="%PARQUET_CXXFLAGS% /WX"
-)
-
-if "%GENERATOR%"=="NMake Makefiles" set need_vcvarsall=1
-
-if defined need_vcvarsall (
- @rem Select desired compiler version
- if "%APPVEYOR_BUILD_WORKER_IMAGE%" == "Visual Studio 2017" (
- call "C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\VC\Auxiliary\Build\vcvarsall.bat" amd64
- ) else (
- call "C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat" amd64
- )
-)
-
-if "%CONFIGURATION%" == "Toolchain" (
- conda install -y boost-cpp=1.63 thrift-cpp=0.11.0 ^
- brotli=1.0.2 zlib=1.2.11 snappy=1.1.7 lz4-c=1.8.0 zstd=1.3.3 ^
- -c conda-forge || exit /B
-
- set ARROW_BUILD_TOOLCHAIN=%MINICONDA%/Library
- set PARQUET_BUILD_TOOLCHAIN=%MINICONDA%/Library
-
- cmake -G "%GENERATOR%" ^
- -DCMAKE_BUILD_TYPE=Release ^
- -DPARQUET_BOOST_USE_SHARED=OFF ^
- -DPARQUET_CXXFLAGS=%PARQUET_CXXFLAGS% ^
- .. || exit /B
-
- cmake --build . --config Release || exit /B
-)
-
-if NOT "%CONFIGURATION%" == "Toolchain" (
- cmake -G "%GENERATOR%" ^
- -DCMAKE_BUILD_TYPE=%CONFIGURATION% ^
- -DPARQUET_BOOST_USE_SHARED=OFF ^
- -DPARQUET_CXXFLAGS=%PARQUET_CXXFLAGS% ^
- -DPARQUET_USE_STATIC_CRT=%PARQUET_USE_STATIC_CRT_OPTION% ^
- .. || exit /B
-
- cmake --build . --config %CONFIGURATION% || exit /B
-)
-
-if NOT "%CONFIGURATION%" == "Debug" (
- @rem Tests are too slow and/or hang in debug mode
- ctest -VV || exit /B
-)
diff --git a/ci/travis_script_cpp.sh b/ci/travis_script_cpp.sh
deleted file mode 100755
index 3031363..0000000
--- a/ci/travis_script_cpp.sh
+++ /dev/null
@@ -1,50 +0,0 @@
-#!/usr/bin/env bash
-
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License. See accompanying LICENSE file.
-
-set -xe
-
-: ${CPP_BUILD_DIR=$TRAVIS_BUILD_DIR/parquet-build}
-
-# Check licenses according to Apache policy
-pushd $TRAVIS_BUILD_DIR
-git archive HEAD -o parquet-cpp-src.tar.gz
-$TRAVIS_BUILD_DIR/dev/release/run-rat.sh parquet-cpp-src.tar.gz
-popd
-
-pushd $CPP_BUILD_DIR
-
-make lint
-
-# PARQUET-626: disabled check for now
-# if [ $TRAVIS_OS_NAME == "linux" ]; then
-# make check-format
-# make check-clang-tidy
-# fi
-
-if [ $TRAVIS_OS_NAME == "linux" ]; then
- make -j4
- ctest -j2 -VV -L unittest
-# Current cpp-coveralls version 0.4 throws an error (PARQUET-1075) on Travis CI. Pin to last working version
- sudo pip install cpp_coveralls==0.3.12
- export PARQUET_ROOT=$TRAVIS_BUILD_DIR
- $TRAVIS_BUILD_DIR/ci/upload_coverage.sh
-else
- make -j4
- BUILD_TYPE=debug
- EXECUTABLE_DIR=$CPP_BUILD_DIR/$BUILD_TYPE
- export LD_LIBRARY_PATH=$EXECUTABLE_DIR:$LD_LIBRARY_PATH
- ctest -j2 -VV -L unittest
-fi
-
-popd
diff --git a/ci/travis_script_static.sh b/ci/travis_script_static.sh
deleted file mode 100755
index 8af574e..0000000
--- a/ci/travis_script_static.sh
+++ /dev/null
@@ -1,90 +0,0 @@
-#!/usr/bin/env bash
-
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License. See accompanying LICENSE file.
-
-set -xe
-
-# Use a C++11 compiler on Linux
-export CC="gcc-4.9"
-export CXX="g++-4.9"
-
-# ----------------------------------------------------------------------
-# Set up external toolchain
-
-MINICONDA_URL="https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh"
-export MINICONDA=$HOME/miniconda
-wget -O miniconda.sh $MINICONDA_URL
-bash miniconda.sh -b -p $MINICONDA
-export PATH="$MINICONDA/bin:$PATH"
-export CPP_TOOLCHAIN=$TRAVIS_BUILD_DIR/cpp-toolchain
-
-conda update -y -q conda
-conda config --set auto_update_conda false
-conda info -a
-
-conda config --set show_channel_urls True
-
-# Help with SSL timeouts to S3
-conda config --set remote_connect_timeout_secs 12
-
-conda info -a
-
-conda create -y -q -p $CPP_TOOLCHAIN \
- boost-cpp thrift-cpp=0.10.0 cmake git \
- -c conda-forge
-
-source activate $CPP_TOOLCHAIN
-
-# ----------------------------------------------------------------------
-
-: ${CPP_BUILD_DIR=$TRAVIS_BUILD_DIR/parquet-build}
-export PARQUET_TEST_DATA=$TRAVIS_BUILD_DIR/data
-export PARQUET_BUILD_TOOLCHAIN=$CPP_TOOLCHAIN
-export LD_LIBRARY_PATH=$CPP_TOOLCHAIN/lib:$LD_LIBRARY_PATH
-export BOOST_ROOT=$CPP_TOOLCHAIN
-export PARQUET_TEST_DATA=$TRAVIS_BUILD_DIR/data
-
-ARROW_EP=$TRAVIS_BUILD_DIR/parquet-build/arrow_ep-prefix/src/arrow_ep-build
-BROTLI_EP=$ARROW_EP/brotli_ep/src/brotli_ep-install/lib/x86_64-linux-gnu
-
-export SNAPPY_STATIC_LIB=$ARROW_EP/snappy_ep/src/snappy_ep-install/lib/libsnappy.a
-export BROTLI_STATIC_LIB_ENC=$BROTLI_EP/libbrotlienc.a
-export BROTLI_STATIC_LIB_DEC=$BROTLI_EP/libbrotlidec.a
-export BROTLI_STATIC_LIB_COMMON=$BROTLI_EP/libbrotlicommon.a
-export ZLIB_STATIC_LIB=$ARROW_EP/zlib_ep/src/zlib_ep-install/lib/libz.a
-export LZ4_STATIC_LIB=$ARROW_EP/lz4_ep-prefix/src/lz4_ep/lib/liblz4.a
-export ZSTD_STATIC_LIB=$ARROW_EP/zstd_ep-prefix/src/zstd_ep/lib/libzstd.a
-
-CMAKE_COMMON_FLAGS="-DPARQUET_BUILD_WARNING_LEVEL=CHECKIN"
-
-if [ $PARQUET_TRAVIS_VALGRIND == "1" ]; then
- CMAKE_COMMON_FLAGS="$CMAKE_COMMON_FLAGS -DPARQUET_TEST_MEMCHECK=ON"
-fi
-
-cmake $CMAKE_COMMON_FLAGS \
- -DPARQUET_CXXFLAGS="$PARQUET_CXXFLAGS" \
- -DPARQUET_ARROW_LINKAGE="static" \
- -DPARQUET_BUILD_SHARED=OFF \
- -DPARQUET_BOOST_USE_SHARED=OFF \
- -DPARQUET_THRIFT_USE_BOOST=ON \
- -DPARQUET_BUILD_BENCHMARKS=ON \
- -DPARQUET_BUILD_EXAMPLES=ON \
- -DPARQUET_GENERATE_COVERAGE=1 \
- $TRAVIS_BUILD_DIR
-
-pushd $CPP_BUILD_DIR
-
-make -j4 VERBOSE=1
-ctest -j2 -VV -L unittest
-
-popd
diff --git a/ci/travis_script_toolchain.sh b/ci/travis_script_toolchain.sh
deleted file mode 100755
index e94552c..0000000
--- a/ci/travis_script_toolchain.sh
+++ /dev/null
@@ -1,88 +0,0 @@
-#!/usr/bin/env bash
-
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License. See accompanying LICENSE file.
-
-set -xe
-
-# Use a C++11 compiler on Linux
-export CC="gcc-4.9"
-export CXX="g++-4.9"
-
-# ----------------------------------------------------------------------
-# Set up external toolchain
-
-MINICONDA_URL="https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh"
-export MINICONDA=$HOME/miniconda
-wget -O miniconda.sh $MINICONDA_URL
-bash miniconda.sh -b -p $MINICONDA
-export PATH="$MINICONDA/bin:$PATH"
-export CPP_TOOLCHAIN=$TRAVIS_BUILD_DIR/cpp-toolchain
-
-conda update -y -q conda
-conda config --set auto_update_conda false
-conda info -a
-
-conda config --set show_channel_urls True
-
-# Help with SSL timeouts to S3
-conda config --set remote_connect_timeout_secs 12
-
-conda info -a
-
-conda create -y -q -p $CPP_TOOLCHAIN \
- boost-cpp thrift-cpp=0.11.0 cmake git \
- -c conda-forge
-
-# ----------------------------------------------------------------------
-
-: ${CPP_BUILD_DIR=$TRAVIS_BUILD_DIR/parquet-build}
-export PARQUET_TEST_DATA=$TRAVIS_BUILD_DIR/data
-export PARQUET_BUILD_TOOLCHAIN=$CPP_TOOLCHAIN
-export LD_LIBRARY_PATH=$CPP_TOOLCHAIN/lib:$LD_LIBRARY_PATH
-export BOOST_ROOT=$CPP_TOOLCHAIN
-
-CMAKE_COMMON_FLAGS="-DPARQUET_BUILD_WARNING_LEVEL=CHECKIN"
-
-if [ $PARQUET_TRAVIS_VALGRIND == "1" ]; then
- CMAKE_COMMON_FLAGS="$CMAKE_COMMON_FLAGS -DPARQUET_TEST_MEMCHECK=ON"
-fi
-
-cmake $CMAKE_COMMON_FLAGS \
- -DPARQUET_CXXFLAGS=-Werror \
- -DPARQUET_GENERATE_COVERAGE=1 \
- -DCMAKE_INSTALL_PREFIX=$CPP_TOOLCHAIN \
- $TRAVIS_BUILD_DIR
-
-pushd $CPP_BUILD_DIR
-
-make -j4
-make install
-ctest -j2 -VV -L unittest
-
-popd
-
-# Build and run the parquet::arrow example. This also tests the usage of parquet-cpp as a library.
-
-pushd $TRAVIS_BUILD_DIR/examples/parquet-arrow
-mkdir build
-pushd build
-
-export ARROW_HOME=$CPP_TOOLCHAIN
-export PARQUET_HOME=$CPP_TOOLCHAIN
-
-cmake ..
-make VERBOSE=1
-./parquet-arrow-reader-writer
-
-popd
-popd
diff --git a/ci/upload_coverage.sh b/ci/upload_coverage.sh
deleted file mode 100755
index 73b37c2..0000000
--- a/ci/upload_coverage.sh
+++ /dev/null
@@ -1,44 +0,0 @@
-#!/usr/bin/env bash
-
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License. See accompanying LICENSE file.
-
-set -e
-set -x
-
-ROOT_DIR=$(cd "$(dirname "$BASH_SOURCE")"; pwd)
-
-mkdir coverage_artifacts
-python $ROOT_DIR/../build-support/collect_coverage.py CMakeFiles/parquet.dir/src/ coverage_artifacts
-
-cd coverage_artifacts
-
-ls -l
-
-echo $PARQUET_ROOT
-
-coveralls --gcov $(which gcov-4.9) \
- --gcov-options '\-l' --root '' \
- --include $PARQUET_ROOT \
- --exclude $PARQUET_ROOT/parquet-build/thirdparty \
- --exclude $PARQUET_ROOT/parquet-build/arrow_ep \
- --exclude $PARQUET_ROOT/parquet-build/brotli_ep \
- --exclude $PARQUET_ROOT/parquet-build/brotli_ep-prefix \
- --exclude $PARQUET_ROOT/parquet-build/gbenchmark_ep \
- --exclude $PARQUET_ROOT/parquet-build/googletest_ep-prefix \
- --exclude $PARQUET_ROOT/parquet-build/snappy_ep \
- --exclude $PARQUET_ROOT/parquet-build/snappy_ep-prefix \
- --exclude $PARQUET_ROOT/parquet-build/zlib_ep \
- --exclude $PARQUET_ROOT/parquet-build/zlib_ep-prefix \
- --exclude $PARQUET_ROOT/build \
- --exclude $PARQUET_ROOT/src/parquet/thrift \
- --exclude /usr
diff --git a/cmake_modules/ArrowExternalProject.cmake b/cmake_modules/ArrowExternalProject.cmake
deleted file mode 100644
index 51eeca5..0000000
--- a/cmake_modules/ArrowExternalProject.cmake
+++ /dev/null
@@ -1,75 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-set(ARROW_PREFIX "${BUILD_OUTPUT_ROOT_DIRECTORY}")
-set(ARROW_INCLUDE_DIR "${ARROW_PREFIX}/include")
-set(ARROW_LIB_DIR "${ARROW_PREFIX}")
-if (MSVC)
- set(ARROW_SHARED_LIB "${ARROW_PREFIX}/bin/arrow.dll")
- set(ARROW_SHARED_IMPLIB "${ARROW_LIB_DIR}/arrow.lib")
- set(ARROW_STATIC_LIB "${ARROW_LIB_DIR}/arrow_static.lib")
-else()
- set(ARROW_SHARED_LIB "${ARROW_LIB_DIR}/libarrow${CMAKE_SHARED_LIBRARY_SUFFIX}")
- set(ARROW_STATIC_LIB "${ARROW_LIB_DIR}/libarrow.a")
-endif()
-
-set(ARROW_CMAKE_ARGS
- -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}
- -DCMAKE_CXX_FLAGS=${EP_CXX_FLAGS}
- -DCMAKE_C_FLAGS=${EP_C_FLAGS}
- -DCMAKE_INSTALL_PREFIX=${ARROW_PREFIX}
- -DCMAKE_INSTALL_LIBDIR=${ARROW_LIB_DIR}
- -DARROW_JEMALLOC=OFF
- -DARROW_IPC=OFF
- -DARROW_WITH_LZ4=ON
- -DARROW_WITH_ZSTD=ON
- -DARROW_BUILD_SHARED=${PARQUET_BUILD_SHARED}
- -DARROW_BOOST_USE_SHARED=${PARQUET_BOOST_USE_SHARED}
- -DARROW_BUILD_TESTS=OFF)
-
-if (MSVC AND PARQUET_USE_STATIC_CRT)
- set(ARROW_CMAKE_ARGS ${ARROW_CMAKE_ARGS} -DARROW_USE_STATIC_CRT=ON)
-endif()
-
-if ("$ENV{PARQUET_ARROW_VERSION}" STREQUAL "")
- # This can be a tag or changeset
- set(ARROW_VERSION "170dc75468efbad2286c630b9103d1aacdb6bada")
-else()
- set(ARROW_VERSION "$ENV{PARQUET_ARROW_VERSION}")
-endif()
-message(STATUS "Building Apache Arrow from commit: ${ARROW_VERSION}")
-
-set(ARROW_URL "https://github.com/apache/arrow/archive/${ARROW_VERSION}.tar.gz")
-
-if (CMAKE_VERSION VERSION_GREATER "3.7")
- set(ARROW_CONFIGURE SOURCE_SUBDIR "cpp" CMAKE_ARGS ${ARROW_CMAKE_ARGS})
-else()
- set(ARROW_CONFIGURE CONFIGURE_COMMAND "${CMAKE_COMMAND}" -G "${CMAKE_GENERATOR}"
- ${ARROW_CMAKE_ARGS} "${CMAKE_CURRENT_BINARY_DIR}/arrow_ep-prefix/src/arrow_ep/cpp")
-endif()
-
-ExternalProject_Add(arrow_ep
- URL ${ARROW_URL}
- ${ARROW_CONFIGURE}
- BUILD_BYPRODUCTS "${ARROW_SHARED_LIB}" "${ARROW_STATIC_LIB}")
-
-if (MSVC)
- ExternalProject_Add_Step(arrow_ep copy_dll_step
- DEPENDEES install
- COMMAND ${CMAKE_COMMAND} -E make_directory ${BUILD_OUTPUT_ROOT_DIRECTORY}
- COMMAND ${CMAKE_COMMAND} -E copy ${ARROW_SHARED_LIB} ${BUILD_OUTPUT_ROOT_DIRECTORY})
-endif()
diff --git a/cmake_modules/BuildUtils.cmake b/cmake_modules/BuildUtils.cmake
deleted file mode 100644
index 1ce891c..0000000
--- a/cmake_modules/BuildUtils.cmake
+++ /dev/null
@@ -1,112 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-function(ADD_LIB LIB_NAME)
- set(options)
- set(one_value_args SHARED_LINK_FLAGS ABI_VERSION SO_VERSION LIB_BUILD_SHARED LIB_BUILD_STATIC)
- set(multi_value_args SOURCES STATIC_LINK_LIBS STATIC_PRIVATE_LINK_LIBS SHARED_LINK_LIBS SHARED_PRIVATE_LINK_LIBS DEPENDENCIES)
- cmake_parse_arguments(ARG "${options}" "${one_value_args}" "${multi_value_args}" ${ARGN})
- if(ARG_UNPARSED_ARGUMENTS)
- message(SEND_ERROR "Error: unrecognized arguments: ${ARG_UNPARSED_ARGUMENTS}")
- endif()
-
- add_library(${LIB_NAME}_objlib OBJECT
- ${ARG_SOURCES})
-
- if (ARG_DEPENDENCIES)
- add_dependencies(${LIB_NAME}_objlib ${ARG_DEPENDENCIES})
- endif()
-
- # Necessary to make static linking into other shared libraries work properly
- set_property(TARGET ${LIB_NAME}_objlib PROPERTY POSITION_INDEPENDENT_CODE 1)
-
- set(RUNTIME_INSTALL_DIR bin)
-
- if (ARG_LIB_BUILD_SHARED)
- add_library(${LIB_NAME}_shared SHARED $<TARGET_OBJECTS:${LIB_NAME}_objlib>)
-
- if(APPLE)
- # On OS X, you can avoid linking at library load time and instead
- # expecting that the symbols have been loaded separately. This happens
- # with libpython* where there can be conflicts between system Python and
- # the Python from a thirdparty distribution
- set(ARG_SHARED_LINK_FLAGS
- "-undefined dynamic_lookup ${ARG_SHARED_LINK_FLAGS}")
- endif()
-
- set_target_properties(${LIB_NAME}_shared
- PROPERTIES
- LIBRARY_OUTPUT_DIRECTORY "${BUILD_OUTPUT_ROOT_DIRECTORY}"
- RUNTIME_OUTPUT_DIRECTORY "${BUILD_OUTPUT_ROOT_DIRECTORY}"
- PDB_OUTPUT_DIRECTORY "${BUILD_OUTPUT_ROOT_DIRECTORY}"
- LINK_FLAGS "${ARG_SHARED_LINK_FLAGS}"
- OUTPUT_NAME ${LIB_NAME}
- VERSION "${ARG_ABI_VERSION}"
- SOVERSION "${ARG_SO_VERSION}")
-
- if (ARG_SHARED_LINK_LIBS OR ARG_SHARED_PRIVATE_LINK_LIBS)
- target_link_libraries(${LIB_NAME}_shared
- LINK_PUBLIC ${ARG_SHARED_LINK_LIBS}
- LINK_PRIVATE ${ARG_SHARED_PRIVATE_LINK_LIBS})
- endif()
-
- if (PARQUET_RPATH_ORIGIN)
- if (APPLE)
- set(_lib_install_rpath "@loader_path")
- else()
- set(_lib_install_rpath "\$ORIGIN")
- endif()
- set_target_properties(${LIB_NAME}_shared PROPERTIES
- INSTALL_RPATH ${_lib_install_rpath})
- elseif (APPLE)
- set_target_properties(${LIB_NAME}_shared
- PROPERTIES
- BUILD_WITH_INSTALL_RPATH ON
- INSTALL_NAME_DIR "@rpath")
- endif()
-
- install(TARGETS ${LIB_NAME}_shared
- RUNTIME DESTINATION ${RUNTIME_INSTALL_DIR}
- LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
- ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR})
- endif()
-
- if (ARG_LIB_BUILD_STATIC)
- if (MSVC)
- set(LIB_NAME_STATIC ${LIB_NAME}_static)
- else()
- set(LIB_NAME_STATIC ${LIB_NAME})
- endif()
- add_library(${LIB_NAME}_static STATIC $<TARGET_OBJECTS:${LIB_NAME}_objlib>)
- set_target_properties(${LIB_NAME}_static
- PROPERTIES
- LIBRARY_OUTPUT_DIRECTORY "${BUILD_OUTPUT_ROOT_DIRECTORY}"
- OUTPUT_NAME ${LIB_NAME_STATIC})
-
- if (ARG_STATIC_LINK_LIBS OR ARG_STATIC_PRIVATE_LINK_LIBS)
- target_link_libraries(${LIB_NAME}_static
- LINK_PUBLIC ${ARG_STATIC_LINK_LIBS}
- LINK_PRIVATE ${ARG_STATIC_PRIVATE_LINK_LIBS})
- endif()
-
- install(TARGETS ${LIB_NAME}_static
- RUNTIME DESTINATION ${RUNTIME_INSTALL_DIR}
- LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
- ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR})
- endif()
-
-endfunction()
diff --git a/cmake_modules/CompilerInfo.cmake b/cmake_modules/CompilerInfo.cmake
deleted file mode 100644
index 654a0d8..0000000
--- a/cmake_modules/CompilerInfo.cmake
+++ /dev/null
@@ -1,63 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-# Sets COMPILER_FAMILY to 'clang' or 'gcc' or 'msvc'
-# Sets COMPILER_VERSION to the version
-if(NOT MSVC)
- set(COMPILER_GET_VERSION_SWITCH "-v")
-endif()
-
-execute_process(COMMAND "${CMAKE_CXX_COMPILER}" ${COMPILER_GET_VERSION_SWITCH}
- OUTPUT_VARIABLE COMPILER_VERSION_FULL
- ERROR_VARIABLE COMPILER_VERSION_FULL)
-message(STATUS "Compiler id: ${CMAKE_CXX_COMPILER_ID}")
-string(TOLOWER "${COMPILER_VERSION_FULL}" COMPILER_VERSION_FULL_LOWER)
-
-if(MSVC)
- set(COMPILER_FAMILY "msvc")
- if ("${COMPILER_VERSION_FULL}" MATCHES ".*Microsoft \\(R\\) C/C\\+\\+ Optimizing Compiler Version 19.*x64")
- string(REGEX REPLACE ".*Optimizing Compiler Version ([0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+).*" "\\1"
- COMPILER_VERSION "${COMPILER_VERSION_FULL}")
- elseif(NOT "${COMPILER_VERSION_FULL}" STREQUAL "")
- message(FATAL_ERROR "Not supported MSVC compiler:\n${COMPILER_VERSION_FULL}\n"
- "Supported MSVC versions: Visual Studio 2015 2017 x64")
- endif()
-
-# clang on Linux and Mac OS X before 10.9
-elseif("${COMPILER_VERSION_FULL}" MATCHES ".*clang version.*")
- set(COMPILER_FAMILY "clang")
- string(REGEX REPLACE ".*clang version ([0-9]+\\.[0-9]+).*" "\\1"
- COMPILER_VERSION "${COMPILER_VERSION_FULL}")
-# clang on Mac OS X 10.9 and later
-elseif("${COMPILER_VERSION_FULL}" MATCHES ".*based on LLVM.*")
- set(COMPILER_FAMILY "clang")
- string(REGEX REPLACE ".*based on LLVM ([0-9]+\\.[0.9]+).*" "\\1"
- COMPILER_VERSION "${COMPILER_VERSION_FULL}")
-
-# clang on Mac OS X, XCode 7+.
-elseif("${COMPILER_VERSION_FULL}" MATCHES ".*clang-.*")
- set(COMPILER_FAMILY "clang")
- set(COMPILER_VERSION "4.0")
-# gcc
-elseif("${COMPILER_VERSION_FULL_LOWER}" MATCHES ".*gcc[ -]version.*")
- set(COMPILER_FAMILY "gcc")
- string(REGEX REPLACE ".*gcc[ -]version ([0-9\\.]+).*" "\\1"
- COMPILER_VERSION "${COMPILER_VERSION_FULL_LOWER}")
-else()
- message(FATAL_ERROR "Unknown compiler. Version info:\n${COMPILER_VERSION_FULL}")
-endif()
-message("Selected compiler ${COMPILER_FAMILY} ${COMPILER_VERSION}")
diff --git a/cmake_modules/FindArrow.cmake b/cmake_modules/FindArrow.cmake
deleted file mode 100644
index b63b1af..0000000
--- a/cmake_modules/FindArrow.cmake
+++ /dev/null
@@ -1,123 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-# - Find ARROW (arrow/api.h, libarrow.a, libarrow.so)
-# This module defines
-# ARROW_INCLUDE_DIR, directory containing headers
-# ARROW_STATIC_LIB, path to libarrow.a
-# ARROW_SHARED_LIB, path to libarrow's shared library
-# ARROW_FOUND, whether arrow has been found
-
-if (DEFINED ENV{ARROW_HOME})
- set(ARROW_HOME "$ENV{ARROW_HOME}")
-endif()
-
-if ("${ARROW_HOME}" STREQUAL "")
- # PARQUET-955. If the user has set $ARROW_HOME in the environment, we respect
- # this, otherwise try to locate the pkgconfig in the system environment
- pkg_check_modules(ARROW arrow)
- if (ARROW_FOUND)
- # We found the pkgconfig
- set(ARROW_INCLUDE_DIR ${ARROW_INCLUDE_DIRS})
-
- if (COMMAND pkg_get_variable)
- pkg_get_variable(ARROW_ABI_VERSION arrow abi_version)
- else()
- set(ARROW_ABI_VERSION "")
- endif()
- if (ARROW_ABI_VERSION STREQUAL "")
- set(ARROW_SHARED_LIB_SUFFIX "")
- else()
- set(ARROW_SHARED_LIB_SUFFIX ".${ARROW_ABI_VERSION}")
- endif()
-
- set(ARROW_LIB_NAME ${CMAKE_SHARED_LIBRARY_PREFIX}arrow)
-
- if (APPLE)
- set(ARROW_SHARED_LIB ${ARROW_LIBDIR}/${ARROW_LIB_NAME}${ARROW_SHARED_LIB_SUFFIX}${CMAKE_SHARED_LIBRARY_SUFFIX})
- else()
- set(ARROW_SHARED_LIB ${ARROW_LIBDIR}/${ARROW_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX}${ARROW_SHARED_LIB_SUFFIX})
- endif()
- set(ARROW_STATIC_LIB ${ARROW_LIBDIR}/${ARROW_LIB_NAME}${CMAKE_STATIC_LIBRARY_SUFFIX})
- endif()
-else()
- set(ARROW_HOME "${ARROW_HOME}")
-
- if (MSVC AND NOT ARROW_MSVC_STATIC_LIB_SUFFIX)
- set(ARROW_MSVC_STATIC_LIB_SUFFIX _static)
- endif()
-
- set(ARROW_SEARCH_HEADER_PATHS
- ${ARROW_HOME}/include
- )
-
- set(ARROW_SEARCH_LIB_PATH
- ${ARROW_HOME}/lib
- )
-
- find_path(ARROW_INCLUDE_DIR arrow/array.h PATHS
- ${ARROW_SEARCH_HEADER_PATHS}
- # make sure we don't accidentally pick up a different version
- NO_DEFAULT_PATH
- )
-
- find_library(ARROW_LIB_PATH NAMES arrow arrow${ARROW_MSVC_STATIC_LIB_SUFFIX}
- PATHS
- ${ARROW_SEARCH_LIB_PATH}
- NO_DEFAULT_PATH)
-
- if (ARROW_INCLUDE_DIR AND (PARQUET_MINIMAL_DEPENDENCY OR ARROW_LIB_PATH))
- set(ARROW_FOUND TRUE)
- set(ARROW_HEADER_NAME arrow/api.h)
- set(ARROW_HEADER ${ARROW_INCLUDE_DIR}/${ARROW_HEADER_NAME})
- set(ARROW_LIB_NAME arrow)
-
- get_filename_component(ARROW_LIBS ${ARROW_LIB_PATH} DIRECTORY)
- set(ARROW_STATIC_LIB ${ARROW_LIBS}/${CMAKE_STATIC_LIBRARY_PREFIX}${ARROW_LIB_NAME}${ARROW_MSVC_STATIC_LIB_SUFFIX}${CMAKE_STATIC_LIBRARY_SUFFIX})
- set(ARROW_SHARED_LIB ${ARROW_LIBS}/${CMAKE_SHARED_LIBRARY_PREFIX}${ARROW_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX})
- set(ARROW_SHARED_IMPLIB ${ARROW_LIBS}/${ARROW_LIB_NAME}.lib)
- endif ()
-endif()
-
-if (ARROW_FOUND)
- if (NOT Arrow_FIND_QUIETLY)
- message(STATUS "Arrow include path: ${ARROW_INCLUDE_DIR}")
- if (PARQUET_MINIMAL_DEPENDENCY)
- message(STATUS "Found the Arrow header: ${ARROW_HEADER}")
- else ()
- message(STATUS "Found the Arrow library: ${ARROW_LIB_PATH}")
- endif ()
- endif ()
-else()
- if (NOT Arrow_FIND_QUIETLY)
- set(ARROW_ERR_MSG "Could not find the Arrow library. Looked for headers")
- set(ARROW_ERR_MSG "${ARROW_ERR_MSG} in ${ARROW_SEARCH_HEADER_PATHS}, and for libs")
- set(ARROW_ERR_MSG "${ARROW_ERR_MSG} in ${ARROW_SEARCH_LIB_PATH}")
- if (Arrow_FIND_REQUIRED)
- message(FATAL_ERROR "${ARROW_ERR_MSG}")
- else (Arrow_FIND_REQUIRED)
- message(STATUS "${ARROW_ERR_MSG}")
- endif (Arrow_FIND_REQUIRED)
- endif ()
-endif()
-
-mark_as_advanced(
- ARROW_FOUND
- ARROW_INCLUDE_DIR
- ARROW_STATIC_LIB
- ARROW_SHARED_LIB
-)
diff --git a/cmake_modules/FindBrotli.cmake b/cmake_modules/FindBrotli.cmake
deleted file mode 100644
index f2e714c..0000000
--- a/cmake_modules/FindBrotli.cmake
+++ /dev/null
@@ -1,116 +0,0 @@
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-# Tries to find Brotli headers and libraries.
-#
-# Usage of this module as follows:
-#
-# find_package(Brotli)
-#
-# Variables used by this module, they can change the default behaviour and need
-# to be set before calling find_package:
-#
-# Brotli_HOME - When set, this path is inspected instead of standard library
-# locations as the root of the Brotli installation.
-# The environment variable BROTLI_HOME overrides this veriable.
-#
-# This module defines
-# BROTLI_INCLUDE_DIR, directory containing headers
-# BROTLI_LIBS, directory containing brotli libraries
-# BROTLI_STATIC_LIB, path to libbrotli.a
-# BROTLI_SHARED_LIB, path to libbrotli's shared library
-# BROTLI_FOUND, whether brotli has been found
-
-if( NOT "${BROTLI_HOME}" STREQUAL "")
- file( TO_CMAKE_PATH "${BROTLI_HOME}" _native_path )
- list( APPEND _brotli_roots ${_native_path} )
-elseif ( Brotli_HOME )
- list( APPEND _brotli_roots ${Brotli_HOME} )
-endif()
-
-find_path( BROTLI_INCLUDE_DIR NAMES brotli/decode.h
- PATHS ${_brotli_roots}
- NO_DEFAULT_PATH
- PATH_SUFFIXES "include" )
-
-find_library( BROTLI_LIBRARY_ENC NAMES libbrotlienc.a brotlienc
- PATHS ${_brotli_roots}
- NO_DEFAULT_PATH
- PATH_SUFFIXES "lib/${CMAKE_LIBRARY_ARCHITECTURE}" "lib" )
-
-find_library( BROTLI_LIBRARY_DEC NAMES libbrotlidec.a brotlidec
- PATHS ${_brotli_roots}
- NO_DEFAULT_PATH
- PATH_SUFFIXES "lib/${CMAKE_LIBRARY_ARCHITECTURE}" "lib" )
-
-find_library( BROTLI_LIBRARY_COMMON NAMES libbrotlicommon.a brotlicommon
- PATHS ${_brotli_roots}
- NO_DEFAULT_PATH
- PATH_SUFFIXES "lib/${CMAKE_LIBRARY_ARCHITECTURE}" "lib" )
-
-set(BROTLI_LIBRARIES ${BROTLI_LIBRARY_ENC} ${BROTLI_LIBRARY_DEC}
- ${BROTLI_LIBRARY_COMMON})
-
-if (BROTLI_INCLUDE_DIR AND (PARQUET_MINIMAL_DEPENDENCY OR BROTLI_LIBRARIES))
- set(BROTLI_FOUND TRUE)
- get_filename_component( BROTLI_LIBS ${BROTLI_LIBRARY_ENC} PATH )
- set(BROTLI_LIB_NAME brotli)
- if (MSVC AND NOT BROTLI_MSVC_STATIC_LIB_SUFFIX)
- set(BROTLI_MSVC_STATIC_LIB_SUFFIX _static)
- endif()
- set(BROTLI_STATIC_LIB
- ${BROTLI_LIBS}/${CMAKE_STATIC_LIBRARY_PREFIX}${BROTLI_LIB_NAME}enc${BROTLI_MSVC_STATIC_LIB_SUFFIX}${CMAKE_STATIC_LIBRARY_SUFFIX}
- ${BROTLI_LIBS}/${CMAKE_STATIC_LIBRARY_PREFIX}${BROTLI_LIB_NAME}dec${BROTLI_MSVC_STATIC_LIB_SUFFIX}${CMAKE_STATIC_LIBRARY_SUFFIX}
- ${BROTLI_LIBS}/${CMAKE_STATIC_LIBRARY_PREFIX}${BROTLI_LIB_NAME}common${BROTLI_MSVC_STATIC_LIB_SUFFIX}${CMAKE_STATIC_LIBRARY_SUFFIX})
- set(BROTLI_STATIC_LIBRARY_ENC ${BROTLI_LIBS}/${CMAKE_STATIC_LIBRARY_PREFIX}${BROTLI_LIB_NAME}enc${BROTLI_MSVC_STATIC_LIB_SUFFIX}${CMAKE_STATIC_LIBRARY_SUFFIX})
- set(BROTLI_STATIC_LIBRARY_DEC ${BROTLI_LIBS}/${CMAKE_STATIC_LIBRARY_PREFIX}${BROTLI_LIB_NAME}dec${BROTLI_MSVC_STATIC_LIB_SUFFIX}${CMAKE_STATIC_LIBRARY_SUFFIX})
- set(BROTLI_STATIC_LIBRARY_COMMON ${BROTLI_LIBS}/${CMAKE_STATIC_LIBRARY_PREFIX}${BROTLI_LIB_NAME}common${BROTLI_MSVC_STATIC_LIB_SUFFIX}${CMAKE_STATIC_LIBRARY_SUFFIX})
- set(BROTLI_SHARED_LIB
- ${BROTLI_LIBS}/${CMAKE_SHARED_LIBRARY_PREFIX}${BROTLI_LIB_NAME}enc${CMAKE_SHARED_LIBRARY_SUFFIX}
- ${BROTLI_LIBS}/${CMAKE_SHARED_LIBRARY_PREFIX}${BROTLI_LIB_NAME}dec${CMAKE_SHARED_LIBRARY_SUFFIX}
- ${BROTLI_LIBS}/${CMAKE_SHARED_LIBRARY_PREFIX}${BROTLI_LIB_NAME}common${CMAKE_SHARED_LIBRARY_SUFFIX})
-else ()
- set(BROTLI_FOUND FALSE)
-endif ()
-
-if (BROTLI_FOUND)
- if (NOT Brotli_FIND_QUIETLY)
- if (PARQUET_MINIMAL_DEPENDENCY)
- message(STATUS "Found the Brotli headers: ${BROTLI_INCLUDE_DIR}")
- else ()
- message(STATUS "Found the Brotli library: ${BROTLI_LIBRARIES}")
- endif ()
- endif ()
-else ()
- if (NOT Brotli_FIND_QUIETLY)
- set(BROTLI_ERR_MSG "Could not find the Brotli library. Looked in ")
- if ( _brotli_roots )
- set(BROTLI_ERR_MSG "${BROTLI_ERR_MSG} in ${_brotli_roots}.")
- else ()
- set(BROTLI_ERR_MSG "${BROTLI_ERR_MSG} system search paths.")
- endif ()
- if (Brotli_FIND_REQUIRED)
- message(FATAL_ERROR "${BROTLI_ERR_MSG}")
- else (Brotli_FIND_REQUIRED)
- message(STATUS "${BROTLI_ERR_MSG}")
- endif (Brotli_FIND_REQUIRED)
- endif ()
-endif ()
-
-mark_as_advanced(
- BROTLI_INCLUDE_DIR
- BROTLI_LIBS
- BROTLI_LIBRARIES
- BROTLI_STATIC_LIB
- BROTLI_SHARED_LIB
-)
diff --git a/cmake_modules/FindClangTools.cmake b/cmake_modules/FindClangTools.cmake
deleted file mode 100644
index 56e2dd7..0000000
--- a/cmake_modules/FindClangTools.cmake
+++ /dev/null
@@ -1,118 +0,0 @@
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-# Tries to find the clang-tidy and clang-format modules
-#
-# Usage of this module as follows:
-#
-# find_package(ClangTools)
-#
-# Variables used by this module, they can change the default behaviour and need
-# to be set before calling find_package:
-#
-# ClangToolsBin_HOME -
-# When set, this path is inspected instead of standard library binary locations
-# to find clang-tidy and clang-format
-#
-# This module defines
-# CLANG_TIDY_BIN, The path to the clang tidy binary
-# CLANG_TIDY_FOUND, Whether clang tidy was found
-# CLANG_FORMAT_BIN, The path to the clang format binary
-# CLANG_TIDY_FOUND, Whether clang format was found
-
-if (DEFINED ENV{HOMEBREW_PREFIX})
- set(HOMEBREW_PREFIX "${ENV{HOMEBREW_PREFIX}")
-else()
- set(HOMEBREW_PREFIX "/usr/local")
-endif()
-
-find_program(CLANG_TIDY_BIN
- NAMES clang-tidy-4.0
- clang-tidy-3.9
- clang-tidy-3.8
- clang-tidy-3.7
- clang-tidy-3.6
- clang-tidy
- PATHS ${ClangTools_PATH} $ENV{CLANG_TOOLS_PATH} /usr/local/bin /usr/bin "${HOMEBREW_PREFIX}/bin"
- NO_DEFAULT_PATH
-)
-
-if ( "${CLANG_TIDY_BIN}" STREQUAL "CLANG_TIDY_BIN-NOTFOUND" )
- set(CLANG_TIDY_FOUND 0)
- message("clang-tidy not found")
-else()
- set(CLANG_TIDY_FOUND 1)
- message("clang-tidy found at ${CLANG_TIDY_BIN}")
-endif()
-
-if (CLANG_FORMAT_VERSION)
- find_program(CLANG_FORMAT_BIN
- NAMES clang-format-${CLANG_FORMAT_VERSION}
- PATHS
- ${ClangTools_PATH}
- $ENV{CLANG_TOOLS_PATH}
- /usr/local/bin /usr/bin "${HOMEBREW_PREFIX}/bin"
- NO_DEFAULT_PATH
- )
-
- # If not found yet, search alternative locations
- if (("${CLANG_FORMAT_BIN}" STREQUAL "CLANG_FORMAT_BIN-NOTFOUND") AND APPLE)
- # Homebrew ships older LLVM versions in /usr/local/opt/llvm@version/
- STRING(REGEX REPLACE "^([0-9]+)\\.[0-9]+" "\\1" CLANG_FORMAT_MAJOR_VERSION "${CLANG_FORMAT_VERSION}")
- STRING(REGEX REPLACE "^[0-9]+\\.([0-9]+)" "\\1" CLANG_FORMAT_MINOR_VERSION "${CLANG_FORMAT_VERSION}")
- if ("${CLANG_FORMAT_MINOR_VERSION}" STREQUAL "0")
- find_program(CLANG_FORMAT_BIN
- NAMES clang-format
- PATHS "${HOMEBREW_PREFIX}/opt/llvm@${CLANG_FORMAT_MAJOR_VERSION}/bin"
- NO_DEFAULT_PATH
- )
- else()
- find_program(CLANG_FORMAT_BIN
- NAMES clang-format
- PATHS "${HOMEBREW_PREFIX}/opt/llvm@${CLANG_FORMAT_VERSION}/bin"
- NO_DEFAULT_PATH
- )
- endif()
-
- if ("${CLANG_FORMAT_BIN}" STREQUAL "CLANG_FORMAT_BIN-NOTFOUND")
- # binary was still not found, look into Cellar
- file(GLOB CLANG_FORMAT_PATH "${HOMEBREW_PREFIX}/Cellar/llvm/${CLANG_FORMAT_VERSION}.*")
- find_program(CLANG_FORMAT_BIN
- NAMES clang-format
- PATHS "${CLANG_FORMAT_PATH}/bin"
- NO_DEFAULT_PATH
- )
- endif()
- endif()
-else()
- find_program(CLANG_FORMAT_BIN
- NAMES clang-format-6.0
- clang-format-5.0
- clang-format-4.0
- clang-format-3.9
- clang-format-3.8
- clang-format-3.7
- clang-format-3.6
- clang-format
- PATHS ${ClangTools_PATH} $ENV{CLANG_TOOLS_PATH} /usr/local/bin /usr/bin "${HOMEBREW_PREFIX}/bin"
- NO_DEFAULT_PATH
- )
-endif()
-
-if ( "${CLANG_FORMAT_BIN}" STREQUAL "CLANG_FORMAT_BIN-NOTFOUND" )
- set(CLANG_FORMAT_FOUND 0)
- message("clang-format not found")
-else()
- set(CLANG_FORMAT_FOUND 1)
- message("clang-format found at ${CLANG_FORMAT_BIN}")
-endif()
diff --git a/cmake_modules/FindGBenchmark.cmake b/cmake_modules/FindGBenchmark.cmake
deleted file mode 100644
index 3e46a60..0000000
--- a/cmake_modules/FindGBenchmark.cmake
+++ /dev/null
@@ -1,88 +0,0 @@
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-# Tries to find Google benchmark headers and libraries.
-#
-# Usage of this module as follows:
-#
-# find_package(GBenchark)
-#
-# Variables used by this module, they can change the default behaviour and need
-# to be set before calling find_package:
-#
-# GBenchmark_HOME - When set, this path is inspected instead of standard library
-# locations as the root of the benchark installation.
-# The environment variable GBENCHMARK_HOME overrides this veriable.
-#
-# This module defines
-# GBENCHMARK_INCLUDE_DIR, directory containing benchmark header directory
-# GBENCHMARK_LIBS, directory containing benchmark libraries
-# GBENCHMARK_STATIC_LIB, path to libbenchmark.a
-# GBENCHMARK_FOUND, whether gbenchmark has been found
-
-if( NOT "$ENV{GBENCHMARK_HOME}" STREQUAL "")
- file( TO_CMAKE_PATH "$ENV{GBENCHMARK_HOME}" _native_path )
- list( APPEND _gbenchmark_roots ${_native_path} )
-elseif ( GBenchmark_HOME )
- list( APPEND _gbenchmark_roots ${GBenchmark_HOME} )
-endif()
-
-# Try the parameterized roots, if they exist
-if ( _gbenchmark_roots )
- find_path( GBENCHMARK_INCLUDE_DIR NAMES benchmark/benchmark.h
- PATHS ${_gbenchmark_roots} NO_DEFAULT_PATH
- PATH_SUFFIXES "include" )
- find_library( GBENCHMARK_LIBRARIES NAMES benchmark
- PATHS ${_gbenchmark_roots} NO_DEFAULT_PATH
- PATH_SUFFIXES "lib" )
-else ()
- find_path( GBENCHMARK_INCLUDE_DIR NAMES benchmark/benchmark.hh )
- find_library( GBENCHMARK_LIBRARIES NAMES benchmark )
-endif ()
-
-
-if (GBENCHMARK_INCLUDE_DIR AND GBENCHMARK_LIBRARIES)
- set(GBENCHMARK_FOUND TRUE)
- get_filename_component( GBENCHMARK_LIBS ${GBENCHMARK_LIBRARIES} PATH )
- set(GBENCHMARK_LIB_NAME libbenchmark)
- set(GBENCHMARK_STATIC_LIB ${GBENCHMARK_LIBS}/${GBENCHMARK_LIB_NAME}.a)
-else ()
- set(GBENCHMARK_FOUND FALSE)
-endif ()
-
-if (GBENCHMARK_FOUND)
- if (NOT GBenchmark_FIND_QUIETLY)
- message(STATUS "Found the GBenchmark library: ${GBENCHMARK_LIBRARIES}")
- endif ()
-else ()
- if (NOT GBenchmark_FIND_QUIETLY)
- set(GBENCHMARK_ERR_MSG "Could not find the GBenchmark library. Looked in ")
- if ( _gbenchmark_roots )
- set(GBENCHMARK_ERR_MSG "${GBENCHMARK_ERR_MSG} in ${_gbenchmark_roots}.")
- else ()
- set(GBENCHMARK_ERR_MSG "${GBENCHMARK_ERR_MSG} system search paths.")
- endif ()
- if (GBenchmark_FIND_REQUIRED)
- message(FATAL_ERROR "${GBENCHMARK_ERR_MSG}")
- else (GBenchmark_FIND_REQUIRED)
- message(STATUS "${GBENCHMARK_ERR_MSG}")
- endif (GBenchmark_FIND_REQUIRED)
- endif ()
-endif ()
-
-mark_as_advanced(
- GBENCHMARK_INCLUDE_DIR
- GBENCHMARK_LIBS
- GBENCHMARK_LIBRARIES
- GBENCHMARK_STATIC_LIB
-)
diff --git a/cmake_modules/FindGTest.cmake b/cmake_modules/FindGTest.cmake
deleted file mode 100644
index c6ea16b..0000000
--- a/cmake_modules/FindGTest.cmake
+++ /dev/null
@@ -1,94 +0,0 @@
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-# Tries to find GTest headers and libraries.
-#
-# Usage of this module as follows:
-#
-# find_package(GTest)
-#
-# Variables used by this module, they can change the default behaviour and need
-# to be set before calling find_package:
-#
-# GTest_HOME - When set, this path is inspected instead of standard library
-# locations as the root of the GTest installation.
-# The environment variable GTEST_HOME overrides this veriable.
-#
-# This module defines
-# GTEST_INCLUDE_DIR, directory containing headers
-# GTEST_LIBS, directory containing gtest libraries
-# GTEST_STATIC_LIB, path to libgtest.a
-# GTEST_MAIN_STATIC_LIB, path to libgtest_main.a
-# GTEST_SHARED_LIB, path to libgtest's shared library
-# GTEST_FOUND, whether gtest has been found
-
-if( NOT "$ENV{GTEST_HOME}" STREQUAL "")
- file( TO_CMAKE_PATH "$ENV{GTEST_HOME}" _native_path )
- list( APPEND _gtest_roots ${_native_path} )
-elseif ( GTest_HOME )
- list( APPEND _gtest_roots ${GTest_HOME} )
-endif()
-
-# Try the parameterized roots, if they exist
-if ( _gtest_roots )
- find_path( GTEST_INCLUDE_DIR NAMES gtest/gtest.h
- PATHS ${_gtest_roots} NO_DEFAULT_PATH
- PATH_SUFFIXES "include" )
- find_library( GTEST_LIBRARIES NAMES gtest gtest_main
- PATHS ${_gtest_roots} NO_DEFAULT_PATH
- PATH_SUFFIXES "lib" )
-else ()
- find_path( GTEST_INCLUDE_DIR NAMES gtest/gtest.h )
- find_library( GTEST_LIBRARIES NAMES gtest )
-endif ()
-
-
-if (GTEST_INCLUDE_DIR AND GTEST_LIBRARIES)
- set(GTEST_FOUND TRUE)
- get_filename_component( GTEST_LIBS ${GTEST_LIBRARIES} PATH )
- set(GTEST_LIB_NAME gtest)
- set(GTEST_STATIC_LIB ${GTEST_LIBS}/${CMAKE_STATIC_LIBRARY_PREFIX}${GTEST_LIB_NAME}${CMAKE_STATIC_LIBRARY_SUFFIX})
- set(GTEST_MAIN_STATIC_LIB ${GTEST_LIBS}/${CMAKE_STATIC_LIBRARY_PREFIX}${GTEST_LIB_NAME}_main${CMAKE_STATIC_LIBRARY_SUFFIX})
- set(GTEST_SHARED_LIB ${GTEST_LIBS}/${CMAKE_SHARED_LIBRARY_PREFIX}${GTEST_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX})
-else ()
- set(GTEST_FOUND FALSE)
-endif ()
-
-if (GTEST_FOUND)
- if (NOT GTest_FIND_QUIETLY)
- message(STATUS "Found the GTest library: ${GTEST_LIBRARIES}")
- endif ()
-else ()
- if (NOT GTest_FIND_QUIETLY)
- set(GTEST_ERR_MSG "Could not find the GTest library. Looked in ")
- if ( _gtest_roots )
- set(GTEST_ERR_MSG "${GTEST_ERR_MSG} in ${_gtest_roots}.")
- else ()
- set(GTEST_ERR_MSG "${GTEST_ERR_MSG} system search paths.")
- endif ()
- if (GTest_FIND_REQUIRED)
- message(FATAL_ERROR "${GTEST_ERR_MSG}")
- else (GTest_FIND_REQUIRED)
- message(STATUS "${GTEST_ERR_MSG}")
- endif (GTest_FIND_REQUIRED)
- endif ()
-endif ()
-
-mark_as_advanced(
- GTEST_INCLUDE_DIR
- GTEST_LIBS
- GTEST_LIBRARIES
- GTEST_STATIC_LIB
- GTEST_MAIN_STATIC_LIB
- GTEST_SHARED_LIB
-)
diff --git a/cmake_modules/FindSnappy.cmake b/cmake_modules/FindSnappy.cmake
deleted file mode 100644
index 867963c..0000000
--- a/cmake_modules/FindSnappy.cmake
+++ /dev/null
@@ -1,94 +0,0 @@
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-# Tries to find Snappy headers and libraries.
-#
-# Usage of this module as follows:
-#
-# find_package(Snappy)
-#
-# Variables used by this module, they can change the default behaviour and need
-# to be set before calling find_package:
-#
-# Snappy_HOME - When set, this path is inspected instead of standard library
-# locations as the root of the Snappy installation.
-# The environment variable SNAPPY_HOME overrides this variable.
-#
-# This module defines
-# SNAPPY_INCLUDE_DIR, directory containing headers
-# SNAPPY_LIBS, directory containing snappy libraries
-# SNAPPY_STATIC_LIB, path to libsnappy.a
-# SNAPPY_SHARED_LIB, path to libsnappy's shared library
-# SNAPPY_FOUND, whether snappy has been found
-
-if( NOT "${SNAPPY_HOME}" STREQUAL "")
- file( TO_CMAKE_PATH "${SNAPPY_HOME}" _native_path )
- list( APPEND _snappy_roots ${_native_path} )
-elseif ( Snappy_HOME )
- list( APPEND _snappy_roots ${Snappy_HOME} )
-endif()
-
-message(STATUS "SNAPPY_HOME: ${SNAPPY_HOME}")
-find_path(SNAPPY_INCLUDE_DIR snappy.h HINTS
- ${_snappy_roots}
- NO_DEFAULT_PATH
- PATH_SUFFIXES "include")
-
-find_library( SNAPPY_LIBRARIES NAMES snappy PATHS
- ${_snappy_roots}
- NO_DEFAULT_PATH
- PATH_SUFFIXES "lib")
-
-if (SNAPPY_INCLUDE_DIR AND (PARQUET_MINIMAL_DEPENDENCY OR SNAPPY_LIBRARIES))
- set(SNAPPY_FOUND TRUE)
- get_filename_component( SNAPPY_LIBS ${SNAPPY_LIBRARIES} PATH )
- set(SNAPPY_HEADER_NAME snappy.h)
- set(SNAPPY_HEADER ${SNAPPY_INCLUDE_DIR}/${SNAPPY_HEADER_NAME})
- set(SNAPPY_LIB_NAME snappy)
- set(SNAPPY_STATIC_LIB ${SNAPPY_LIBS}/${CMAKE_STATIC_LIBRARY_PREFIX}${SNAPPY_LIB_NAME}${SNAPPY_MSVC_STATIC_LIB_SUFFIX}${CMAKE_STATIC_LIBRARY_SUFFIX})
- set(SNAPPY_SHARED_LIB ${SNAPPY_LIBS}/${CMAKE_SHARED_LIBRARY_PREFIX}${SNAPPY_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX})
-else ()
- set(SNAPPY_FOUND FALSE)
-endif ()
-
-if (SNAPPY_FOUND)
- if (NOT Snappy_FIND_QUIETLY)
- if (PARQUET_MINIMAL_DEPENDENCY)
- message(STATUS "Found the Snappy header: ${SNAPPY_HEADER}")
- else ()
- message(STATUS "Found the Snappy library: ${SNAPPY_LIBRARIES}")
- endif ()
- endif ()
-else ()
- if (NOT Snappy_FIND_QUIETLY)
- set(SNAPPY_ERR_MSG "Could not find the Snappy library. Looked in ")
- if ( _snappy_roots )
- set(SNAPPY_ERR_MSG "${SNAPPY_ERR_MSG} in ${_snappy_roots}.")
- else ()
- set(SNAPPY_ERR_MSG "${SNAPPY_ERR_MSG} system search paths.")
- endif ()
- if (Snappy_FIND_REQUIRED)
- message(FATAL_ERROR "${SNAPPY_ERR_MSG}")
- else (Snappy_FIND_REQUIRED)
- message(STATUS "${SNAPPY_ERR_MSG}")
- endif (Snappy_FIND_REQUIRED)
- endif ()
-endif ()
-
-mark_as_advanced(
- SNAPPY_INCLUDE_DIR
- SNAPPY_LIBS
- SNAPPY_LIBRARIES
- SNAPPY_STATIC_LIB
- SNAPPY_SHARED_LIB
-)
diff --git a/cmake_modules/FindThrift.cmake b/cmake_modules/FindThrift.cmake
deleted file mode 100644
index 25f5082..0000000
--- a/cmake_modules/FindThrift.cmake
+++ /dev/null
@@ -1,102 +0,0 @@
-# Copyright 2012 Cloudera Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# - Find Thrift (a cross platform RPC lib/tool)
-#
-# Variables used by this module, they can change the default behaviour and need
-# to be set before calling find_package:
-#
-# Thrift_HOME - When set, this path is inspected instead of standard library
-# locations as the root of the Thrift installation.
-# The environment variable THRIFT_HOME overrides this variable.
-#
-# This module defines
-# THRIFT_VERSION, version string of ant if found
-# THRIFT_INCLUDE_DIR, where to find THRIFT headers
-# THRIFT_CONTRIB_DIR, where contrib thrift files (e.g. fb303.thrift) are installed
-# THRIFT_STATIC_LIB, THRIFT static library
-# THRIFT_FOUND, If false, do not try to use ant
-
-# prefer the thrift version supplied in THRIFT_HOME
-if( NOT "${THRIFT_HOME}" STREQUAL "")
- file( TO_CMAKE_PATH "${THRIFT_HOME}" _native_path )
- list( APPEND _thrift_roots ${_native_path} )
-elseif ( Thrift_HOME )
- list( APPEND _thrift_roots ${Thrift_HOME} )
-endif()
-
-message(STATUS "THRIFT_HOME: ${THRIFT_HOME}")
-find_path(THRIFT_INCLUDE_DIR thrift/Thrift.h HINTS
- ${_thrift_roots}
- NO_DEFAULT_PATH
- PATH_SUFFIXES "include"
-)
-
-find_path(THRIFT_CONTRIB_DIR share/fb303/if/fb303.thrift HINTS
- ${_thrift_roots}
- NO_DEFAULT_PATH
-)
-
-if (MSVC AND NOT THRIFT_MSVC_STATIC_LIB_SUFFIX)
- set(THRIFT_MSVC_STATIC_LIB_SUFFIX md)
-endif()
-
-find_library(THRIFT_STATIC_LIB NAMES
- ${CMAKE_STATIC_LIBRARY_PREFIX}thrift${THRIFT_MSVC_STATIC_LIB_SUFFIX}${CMAKE_STATIC_LIBRARY_SUFFIX}
- HINTS ${_thrift_roots}
- NO_DEFAULT_PATH
- PATH_SUFFIXES "lib/${CMAKE_LIBRARY_ARCHITECTURE}" "lib"
-)
-
-find_program(THRIFT_COMPILER thrift HINTS
- ${_thrift_roots}
- NO_DEFAULT_PATH
- PATH_SUFFIXES "bin"
-)
-
-if (THRIFT_STATIC_LIB)
- set(THRIFT_FOUND TRUE)
- exec_program(${THRIFT_COMPILER}
- ARGS -version OUTPUT_VARIABLE THRIFT_VERSION RETURN_VALUE THRIFT_RETURN)
-else ()
- set(THRIFT_FOUND FALSE)
-endif ()
-
-if (THRIFT_FOUND)
- if (NOT Thrift_FIND_QUIETLY)
- message(STATUS "Thrift version: ${THRIFT_VERSION}")
- endif ()
-else ()
- if (NOT Thrift_FIND_QUIETLY)
- set(THRIFT_ERR_MSG "Thrift compiler/libraries NOT found: ${THRIFT_RETURN}")
- set(THRIFT_ERR_MSG "${THRIFT_ERR_MSG} (${THRIFT_INCLUDE_DIR}, ${THRIFT_STATIC_LIB}).")
- if ( _thrift_roots )
- set(THRIFT_ERR_MSG "${THRIFT_ERR_MSG} Looked in ${_thrift_roots}.")
- else ()
- set(THRIFT_ERR_MSG "${THRIFT_ERR_MSG} Looked in system search paths.")
- endif ()
- if ( Thrift_FIND_REQUIRED )
- message(FATAL_ERROR "${THRIFT_ERR_MSG}")
- else ()
- message(STATUS "${THRIFT_ERR_MSG}")
- endif ()
- endif ()
-endif ()
-
-
-mark_as_advanced(
- THRIFT_STATIC_LIB
- THRIFT_COMPILER
- THRIFT_INCLUDE_DIR
-)
diff --git a/cmake_modules/FindZLIB.cmake b/cmake_modules/FindZLIB.cmake
deleted file mode 100644
index 78b84f2..0000000
--- a/cmake_modules/FindZLIB.cmake
+++ /dev/null
@@ -1,105 +0,0 @@
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-# Tries to find ZLIB headers and libraries.
-#
-# Usage of this module as follows:
-#
-# find_package(ZLIB)
-#
-# Variables used by this module, they can change the default behaviour and need
-# to be set before calling find_package:
-#
-# ZLIB_HOME - When set, this path is inspected instead of standard library
-# locations as the root of the ZLIB installation.
-# The environment variable ZLIB_HOME overrides this variable.
-#
-# - Find ZLIB (zlib.h, libz.a, libz.so, and libz.so.1)
-# This module defines
-# ZLIB_INCLUDE_DIR, directory containing headers
-# ZLIB_LIBS, directory containing zlib libraries
-# ZLIB_STATIC_LIB, path to libz.a
-# ZLIB_SHARED_LIB, path to libz's shared library
-# ZLIB_FOUND, whether zlib has been found
-
-if( NOT "${ZLIB_HOME}" STREQUAL "")
- file( TO_CMAKE_PATH "${ZLIB_HOME}" _native_path )
- list( APPEND _zlib_roots ${_native_path} )
-elseif ( ZLIB_HOME )
- list( APPEND _zlib_roots ${ZLIB_HOME} )
-endif()
-
-# Try the parameterized roots, if they exist
-if ( _zlib_roots )
- find_path( ZLIB_INCLUDE_DIR NAMES zlib.h
- PATHS ${_zlib_roots} NO_DEFAULT_PATH
- PATH_SUFFIXES "include" )
- find_library( ZLIB_LIBRARIES NAMES libz.a zlib
- PATHS ${_zlib_roots} NO_DEFAULT_PATH
- PATH_SUFFIXES "lib" )
-else ()
- find_path( ZLIB_INCLUDE_DIR NAMES zlib.h )
- # Only look for the static library
- find_library( ZLIB_LIBRARIES NAMES libz.a zlib )
-endif ()
-
-
-if (ZLIB_INCLUDE_DIR AND (PARQUET_MINIMAL_DEPENDENCY OR ZLIB_LIBRARIES))
- set(ZLIB_FOUND TRUE)
- get_filename_component( ZLIB_LIBS ${ZLIB_LIBRARIES} PATH )
- set(ZLIB_HEADER_NAME zlib.h)
- set(ZLIB_HEADER ${ZLIB_INCLUDE_DIR}/${ZLIB_HEADER_NAME})
- set(ZLIB_LIB_NAME z)
- if (MSVC)
- if (NOT ZLIB_MSVC_STATIC_LIB_SUFFIX)
- set(ZLIB_MSVC_STATIC_LIB_SUFFIX libstatic)
- endif()
- set(ZLIB_MSVC_SHARED_LIB_SUFFIX lib)
- endif()
- set(ZLIB_STATIC_LIB ${ZLIB_LIBS}/${CMAKE_STATIC_LIBRARY_PREFIX}${ZLIB_LIB_NAME}${ZLIB_MSVC_STATIC_LIB_SUFFIX}${CMAKE_STATIC_LIBRARY_SUFFIX})
- set(ZLIB_SHARED_LIB ${ZLIB_LIBS}/${CMAKE_SHARED_LIBRARY_PREFIX}${ZLIB_LIB_NAME}${ZLIB_MSVC_SHARED_LIB_SUFFIX}${CMAKE_SHARED_LIBRARY_SUFFIX})
-else ()
- set(ZLIB_FOUND FALSE)
-endif ()
-
-if (ZLIB_FOUND)
- if (NOT ZLIB_FIND_QUIETLY)
- if (PARQUET_MINIMAL_DEPENDENCY)
- message(STATUS "Found the ZLIB header: ${ZLIB_HEADER}")
- else()
- message(STATUS "Found the ZLIB library: ${ZLIB_LIBRARIES}")
- endif ()
- endif ()
-else ()
- if (NOT ZLIB_FIND_QUIETLY)
- set(ZLIB_ERR_MSG "Could not find the ZLIB library. Looked in ")
- if ( _zlib_roots )
- set(ZLIB_ERR_MSG "${ZLIB_ERR_MSG} in ${_zlib_roots}.")
- else ()
- set(ZLIB_ERR_MSG "${ZLIB_ERR_MSG} system search paths.")
- endif ()
- if (ZLIB_FIND_REQUIRED)
- message(FATAL_ERROR "${ZLIB_ERR_MSG}")
- else (ZLIB_FIND_REQUIRED)
- message(STATUS "${ZLIB_ERR_MSG}")
- endif (ZLIB_FIND_REQUIRED)
- endif ()
-endif ()
-
-mark_as_advanced(
- ZLIB_INCLUDE_DIR
- ZLIB_LIBS
- ZLIB_LIBRARIES
- ZLIB_STATIC_LIB
- ZLIB_SHARED_LIB
-)
diff --git a/cmake_modules/SetupCxxFlags.cmake b/cmake_modules/SetupCxxFlags.cmake
deleted file mode 100644
index 0029501..0000000
--- a/cmake_modules/SetupCxxFlags.cmake
+++ /dev/null
@@ -1,248 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-# Check if the target architecture and compiler supports some special
-# instruction sets that would boost performance.
-include(CheckCXXCompilerFlag)
-# x86/amd64 compiler flags
-CHECK_CXX_COMPILER_FLAG("-msse3" CXX_SUPPORTS_SSE3)
-# power compiler flags
-CHECK_CXX_COMPILER_FLAG("-maltivec" CXX_SUPPORTS_ALTIVEC)
-
-# compiler flags that are common across debug/release builds
-
-if (MSVC)
- # TODO(wesm): Change usages of C runtime functions that MSVC says are
- # insecure, like std::getenv
- add_definitions(-D_CRT_SECURE_NO_WARNINGS)
-
- # Use __declspec(dllexport) during library build, other users of the Parquet
- # headers will see dllimport
- add_definitions(-DPARQUET_EXPORTING)
-
- if (CMAKE_CXX_COMPILER_ID STREQUAL "Clang")
- # clang-cl
- set(CXX_COMMON_FLAGS "-EHsc")
- elseif(${CMAKE_CXX_COMPILER_VERSION} VERSION_LESS 19)
- message(FATAL_ERROR "Only MSVC 2015 (Version 19.0) and later are supported
- by Parquet. Found version ${CMAKE_CXX_COMPILER_VERSION}.")
- else()
- # Fix annoying D9025 warning
- string(REPLACE "/W3" "" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
-
- # Set desired warning level (e.g. set /W4 for more warnings)
- set(CXX_COMMON_FLAGS "/W3")
- endif()
-
- if (PARQUET_USE_STATIC_CRT)
- foreach (c_flag CMAKE_CXX_FLAGS CMAKE_CXX_FLAGS_RELEASE CMAKE_CXX_FLAGS_DEBUG
- CMAKE_CXX_FLAGS_MINSIZEREL CMAKE_CXX_FLAGS_RELWITHDEBINFO
- CMAKE_C_FLAGS CMAKE_C_FLAGS_RELEASE CMAKE_C_FLAGS_DEBUG
- CMAKE_C_FLAGS_MINSIZEREL CMAKE_C_FLAGS_RELWITHDEBINFO)
- string(REPLACE "/MD" "-MT" ${c_flag} "${${c_flag}}")
- endforeach()
- endif()
-else()
- # Common flags set below with warning level
- set(CXX_COMMON_FLAGS "")
-endif()
-
-# Build warning level (CHECKIN, EVERYTHING, etc.)
-
-# if no build warning level is specified, default to development warning level
-if (NOT PARQUET_BUILD_WARNING_LEVEL)
- set(PARQUET_BUILD_WARNING_LEVEL Production)
-endif(NOT PARQUET_BUILD_WARNING_LEVEL)
-
-string(TOUPPER ${PARQUET_BUILD_WARNING_LEVEL} UPPERCASE_BUILD_WARNING_LEVEL)
-
-if ("${UPPERCASE_BUILD_WARNING_LEVEL}" STREQUAL "CHECKIN")
- # Pre-checkin builds
- if ("${COMPILER_FAMILY}" STREQUAL "msvc")
- string(REPLACE "/W3" "" CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS}")
- set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} /W3")
- # Treat all compiler warnings as errors
- set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} /WX")
- elseif ("${COMPILER_FAMILY}" STREQUAL "clang")
- set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -Weverything -Wno-c++98-compat \
--Wno-c++98-compat-pedantic -Wno-deprecated -Wno-deprecated-declarations \
--Wno-weak-vtables -Wno-padded -Wno-comma -Wno-unused-parameter -Wno-undef \
--Wno-shadow -Wno-switch-enum -Wno-exit-time-destructors \
--Wno-global-constructors -Wno-weak-template-vtables -Wno-undefined-reinterpret-cast \
--Wno-implicit-fallthrough -Wno-unreachable-code-return \
--Wno-documentation-deprecated-sync \
--Wno-float-equal -Wno-missing-prototypes \
--Wno-old-style-cast -Wno-covered-switch-default \
--Wno-format-nonliteral -Wno-missing-noreturn \
--Wno-cast-align -Wno-vla-extension -Wno-shift-sign-overflow \
--Wno-used-but-marked-unused -Wno-missing-variable-declarations \
--Wno-gnu-zero-variadic-macro-arguments -Wconversion -Wno-sign-conversion \
--Wno-disabled-macro-expansion -Wno-shorten-64-to-32")
-
- message(STATUS "Clang version: ${COMPILER_VERSION}")
-
- # Version numbers where warnings are introduced
- if ("${COMPILER_VERSION}" VERSION_GREATER "3.3")
- set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -Wno-gnu-folding-constant")
- endif()
- if ("${COMPILER_VERSION}" VERSION_GREATER "3.6")
- set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -Wno-reserved-id-macro")
- set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -Wno-range-loop-analysis")
- endif()
- if ("${COMPILER_VERSION}" VERSION_GREATER "3.7")
- set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -Wno-double-promotion")
- endif()
- if ("${COMPILER_VERSION}" VERSION_GREATER "3.8")
- set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -Wno-undefined-func-template")
- endif()
-
- if ("${COMPILER_VERSION}" VERSION_GREATER "4.0")
- set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -Wno-unused-template \
--Wno-zero-as-null-pointer-constant")
- endif()
-
- # Treat all compiler warnings as errors
- set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -Wno-unknown-warning-option -Werror")
- elseif ("${COMPILER_FAMILY}" STREQUAL "gcc")
- set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -Wall \
--Wno-deprecated-declarations -Wconversion -Wno-sign-conversion")
- # Treat all compiler warnings as errors
- set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -Werror")
- else()
- message(FATAL_ERROR "Unknown compiler. Version info:\n${COMPILER_VERSION_FULL}")
- endif()
-elseif ("${UPPERCASE_BUILD_WARNING_LEVEL}" STREQUAL "EVERYTHING")
- # Pedantic builds for fixing warnings
- if ("${COMPILER_FAMILY}" STREQUAL "msvc")
- string(REPLACE "/W3" "" CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS}")
- set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} /Wall")
- # https://docs.microsoft.com/en-us/cpp/build/reference/compiler-option-warning-level
- # /wdnnnn disables a warning where "nnnn" is a warning number
- # Treat all compiler warnings as errors
- set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} /WX")
- elseif ("${COMPILER_FAMILY}" STREQUAL "clang")
- set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -Weverything -Wno-c++98-compat -Wno-c++98-compat-pedantic")
- # Treat all compiler warnings as errors
- set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -Werror")
- elseif ("${COMPILER_FAMILY}" STREQUAL "gcc")
- set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -Wall -Wpedantic -Wextra -Wno-unused-parameter")
- # Treat all compiler warnings as errors
- set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -Werror")
- else()
- message(FATAL_ERROR "Unknown compiler. Version info:\n${COMPILER_VERSION_FULL}")
- endif()
-else()
- # Production builds (warning are not treated as errors)
- if ("${COMPILER_FAMILY}" STREQUAL "msvc")
- # https://docs.microsoft.com/en-us/cpp/build/reference/compiler-option-warning-level
- # TODO: Enable /Wall and disable individual warnings until build compiles without errors
- # /wdnnnn disables a warning where "nnnn" is a warning number
- string(REPLACE "/W3" "" CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS}")
- set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} /W3")
- elseif ("${COMPILER_FAMILY}" STREQUAL "clang")
- set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -Wall")
- elseif ("${COMPILER_FAMILY}" STREQUAL "gcc")
- set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -Wall")
- else()
- message(FATAL_ERROR "Unknown compiler. Version info:\n${COMPILER_VERSION_FULL}")
- endif()
-endif()
-
-# if build warning flags is set, add to CXX_COMMON_FLAGS
-if (BUILD_WARNING_FLAGS)
- # Use BUILD_WARNING_FLAGS with BUILD_WARNING_LEVEL=everything to disable
- # warnings (use with Clang's -Weverything flag to find potential errors)
- set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} ${BUILD_WARNING_FLAGS}")
-endif(BUILD_WARNING_FLAGS)
-
-if ("${COMPILER_FAMILY}" STREQUAL "clang")
- # Using Clang with ccache causes a bunch of spurious warnings that are
- # purportedly fixed in the next version of ccache. See the following for details:
- #
- # http://petereisentraut.blogspot.com/2011/05/ccache-and-clang.html
- # http://petereisentraut.blogspot.com/2011/09/ccache-and-clang-part-2.html
- set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Qunused-arguments")
- set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${CMAKE_CLANG_OPTIONS}")
-endif()
-
-if (NOT ("${COMPILER_FAMILY}" STREQUAL "msvc"))
-set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -std=c++11")
-endif()
-
-if ("${COMPILER_FAMILY}" STREQUAL "msvc")
- # Support large object code
- set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} /bigobj")
-
- # MSVC version of -Wno-deprecated
- set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} /wd4996")
-endif()
-
-if (PARQUET_USE_SSE)
- SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -march=native")
- add_definitions(-DPARQUET_USE_SSE)
-endif()
-
-if (APPLE)
- # Depending on the default OSX_DEPLOYMENT_TARGET (< 10.9), libstdc++ may be
- # the default standard library which does not support C++11. libc++ is the
- # default from 10.9 onward.
- set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -stdlib=libc++")
-endif()
-
-# compiler flags for different build types (run 'cmake -DCMAKE_BUILD_TYPE=<type> .')
-# For all builds:
-# For CMAKE_BUILD_TYPE=Debug
-# -ggdb: Enable gdb debugging
-# For CMAKE_BUILD_TYPE=FastDebug
-# Same as DEBUG, except with some optimizations on.
-# For CMAKE_BUILD_TYPE=Release
-# -O3: Enable all compiler optimizations
-# Debug symbols are stripped for reduced binary size. Add
-# -DPARQUET_CXXFLAGS="-g" to add them
-if (NOT MSVC)
- set(CXX_FLAGS_DEBUG "-ggdb -O0")
- set(CXX_FLAGS_FASTDEBUG "-ggdb -O1")
- set(CXX_FLAGS_RELEASE "-O3 -DNDEBUG")
-endif()
-
-set(CXX_FLAGS_PROFILE_GEN "${CXX_FLAGS_RELEASE} -fprofile-generate")
-set(CXX_FLAGS_PROFILE_BUILD "${CXX_FLAGS_RELEASE} -fprofile-use")
-
-# if no build build type is specified, default to debug builds
-if (NOT CMAKE_BUILD_TYPE)
- set(CMAKE_BUILD_TYPE Debug)
-endif(NOT CMAKE_BUILD_TYPE)
-
-string (TOUPPER ${CMAKE_BUILD_TYPE} CMAKE_BUILD_TYPE)
-
-# Set compile flags based on the build type.
-message("Configured for ${CMAKE_BUILD_TYPE} build (set with cmake -DCMAKE_BUILD_TYPE={release,debug,...})")
-if ("${CMAKE_BUILD_TYPE}" STREQUAL "DEBUG")
- set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${CXX_FLAGS_DEBUG}")
-elseif ("${CMAKE_BUILD_TYPE}" STREQUAL "FASTDEBUG")
- set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${CXX_FLAGS_FASTDEBUG}")
-elseif ("${CMAKE_BUILD_TYPE}" STREQUAL "RELEASE")
- set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${CXX_FLAGS_RELEASE}")
-elseif ("${CMAKE_BUILD_TYPE}" STREQUAL "PROFILE_GEN")
- set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${CXX_FLAGS_PROFILE_GEN}")
-elseif ("${CMAKE_BUILD_TYPE}" STREQUAL "PROFILE_BUILD")
- set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${CXX_FLAGS_PROFILE_BUILD}")
-else()
- message(FATAL_ERROR "Unknown build type: ${CMAKE_BUILD_TYPE}")
-endif ()
-
-message(STATUS "Build Type: ${CMAKE_BUILD_TYPE}")
diff --git a/cmake_modules/SnappyCMakeLists.txt b/cmake_modules/SnappyCMakeLists.txt
deleted file mode 100644
index 0348eb1..0000000
--- a/cmake_modules/SnappyCMakeLists.txt
+++ /dev/null
@@ -1,99 +0,0 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-# Originally import from the Snappy repository with the following license:
-#
-# Copyright 2008 Google Inc. All Rights Reserved.
-#
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions are
-# met:
-#
-# * Redistributions of source code must retain the above copyright
-# notice, this list of conditions and the following disclaimer.
-# * Redistributions in binary form must reproduce the above
-# copyright notice, this list of conditions and the following disclaimer
-# in the documentation and/or other materials provided with the
-# distribution.
-# * Neither the name of Google Inc. nor the names of its
-# contributors may be used to endorse or promote products derived from
-# this software without specific prior written permission.
-#
-# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-CMAKE_MINIMUM_REQUIRED(VERSION 2.6)
-PROJECT(snappy)
-
-INCLUDE(CheckIncludeFiles)
-INCLUDE(CMakePackageConfigHelpers)
-
-CHECK_INCLUDE_FILES("stdint.h" HAVE_STDINT_H)
-CHECK_INCLUDE_FILES("stddef.h" HAVE_STDDEF_H)
-CHECK_INCLUDE_FILES("sys/uio.h" HAVE_SYS_UIO_H)
-
-if (NOT HAVE_SYS_UIO_H)
- set(HAVE_SYS_UIO_H 0)
-endif()
-
-if (NOT HAVE_STDINT_H)
- set(HAVE_STDINT_H 0)
-endif()
-
-if (NOT HAVE_STDDEF_H)
- set(HAVE_STDDEF_H 0)
-endif()
-
-set(ac_cv_have_stdint_h ${HAVE_STDINT_H})
-set(ac_cv_have_stddef_h ${HAVE_STDDEF_H})
-set(ac_cv_have_sys_uio_h ${HAVE_SYS_UIO_H})
-CONFIGURE_FILE(${snappy_SOURCE_DIR}/snappy-stubs-public.h.in
- snappy-stubs-public.h)
-
-if (WIN32)
- ADD_DEFINITIONS(-D_CRT_SECURE_NO_WARNINGS)
-endif()
-
-set(SNAPPY_SRCS snappy.cc
- snappy-c.cc
- snappy-stubs-internal.cc
- snappy-sinksource.cc
- snappy.h
- snappy-c.h
- snappy-sinksource.h
- snappy-stubs-public.h)
-
-add_library(snappy SHARED ${SNAPPY_SRCS})
-add_library(snappystatic STATIC ${SNAPPY_SRCS})
-
-TARGET_COMPILE_DEFINITIONS(snappy PRIVATE -DHAVE_CONFIG_H)
-TARGET_COMPILE_DEFINITIONS(snappystatic PRIVATE -DHAVE_CONFIG_H)
-
-install(FILES snappy.h
- snappy-c.h
- snappy-sinksource.h
- ${snappy_BINARY_DIR}/snappy-stubs-public.h
- DESTINATION include)
-
-install(TARGETS snappy snappystatic
- RUNTIME DESTINATION bin
- LIBRARY DESTINATION lib
- ARCHIVE DESTINATION lib)
diff --git a/cmake_modules/SnappyConfig.h b/cmake_modules/SnappyConfig.h
deleted file mode 100644
index 5764deb..0000000
--- a/cmake_modules/SnappyConfig.h
+++ /dev/null
@@ -1,55 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-//
-// Originally import from the Snappy repository with the following license:
-//
-// Copyright 2008 Google Inc. All Rights Reserved.
-//
-// Redistribution and use in source and binary forms, with or without
-// modification, are permitted provided that the following conditions are
-// met:
-//
-// * Redistributions of source code must retain the above copyright
-// notice, this list of conditions and the following disclaimer.
-// * Redistributions in binary form must reproduce the above
-// copyright notice, this list of conditions and the following disclaimer
-// in the documentation and/or other materials provided with the
-// distribution.
-// * Neither the name of Google Inc. nor the names of its
-// contributors may be used to endorse or promote products derived from
-// this software without specific prior written permission.
-//
-// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-#ifndef SNAPPY_CONFIG_H
-#define SNAPPY_CONFIG_H 1
-
-#if defined(_MSC_VER) && (_MSC_VER <= 1900)
-typedef __int64 ssize_t;
-#endif
-
-#endif
diff --git a/cmake_modules/ThirdpartyToolchain.cmake b/cmake_modules/ThirdpartyToolchain.cmake
deleted file mode 100644
index 1934742..0000000
--- a/cmake_modules/ThirdpartyToolchain.cmake
+++ /dev/null
@@ -1,419 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-set(GTEST_VERSION "1.8.0")
-set(GBENCHMARK_VERSION "1.1.0")
-set(THRIFT_VENDOR_VERSION "0.11.0")
-
-string(TOUPPER ${CMAKE_BUILD_TYPE} UPPERCASE_BUILD_TYPE)
-
-set(EP_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${CMAKE_CXX_FLAGS_${UPPERCASE_BUILD_TYPE}}")
-set(EP_C_FLAGS "${CMAKE_C_FLAGS} ${CMAKE_C_FLAGS_${UPPERCASE_BUILD_TYPE}}")
-
-if (NOT MSVC)
- # Set -fPIC on all external projects
- set(EP_CXX_FLAGS "${EP_CXX_FLAGS} -fPIC")
- set(EP_C_FLAGS "${EP_C_FLAGS} -fPIC")
-endif()
-
-if (NOT PARQUET_VERBOSE_THIRDPARTY_BUILD)
- set(EP_LOG_OPTIONS
- LOG_CONFIGURE 1
- LOG_BUILD 1
- LOG_INSTALL 1
- LOG_DOWNLOAD 1)
- set(Boost_DEBUG FALSE)
-else()
- set(EP_LOG_OPTIONS)
- set(Boost_DEBUG TRUE)
-endif()
-
-# ----------------------------------------------------------------------
-# Configure toolchain with environment variables, if the exist
-
-if (NOT "$ENV{PARQUET_BUILD_TOOLCHAIN}" STREQUAL "")
- set(THRIFT_HOME "$ENV{PARQUET_BUILD_TOOLCHAIN}")
- set(ARROW_HOME "$ENV{PARQUET_BUILD_TOOLCHAIN}")
-
- if (NOT DEFINED ENV{BOOST_ROOT})
- # Since we have to set this in the environment, we check whether
- # $BOOST_ROOT is defined inside here
- set(ENV{BOOST_ROOT} "$ENV{PARQUET_BUILD_TOOLCHAIN}")
- endif()
-endif()
-
-if (DEFINED ENV{THRIFT_HOME})
- set(THRIFT_HOME "$ENV{THRIFT_HOME}")
-endif()
-
-# ----------------------------------------------------------------------
-# Boost
-
-# find boost headers and libs
-set(Boost_USE_MULTITHREADED ON)
-if (MSVC AND PARQUET_USE_STATIC_CRT)
- set(Boost_USE_STATIC_RUNTIME ON)
-endif()
-if (MSVC)
- # disable autolinking in boost
- add_definitions(-DBOOST_ALL_NO_LIB)
-endif()
-if (PARQUET_BOOST_USE_SHARED)
- # Find shared Boost libraries.
- set(Boost_USE_STATIC_LIBS OFF)
-
- if (MSVC)
- # force all boost libraries to dynamic link
- add_definitions(-DBOOST_ALL_DYN_LINK)
- find_package(Boost COMPONENTS regex system REQUIRED)
- else()
- find_package(Boost COMPONENTS regex REQUIRED)
- endif()
-
- if ("${UPPERCASE_BUILD_TYPE}" STREQUAL "DEBUG")
- set(BOOST_SHARED_REGEX_LIBRARY ${Boost_REGEX_LIBRARY_DEBUG})
- if (MSVC)
- set(BOOST_SHARED_SYSTEM_LIBRARY ${Boost_SYSTEM_LIBRARY_DEBUG})
- endif()
- else()
- set(BOOST_SHARED_REGEX_LIBRARY ${Boost_REGEX_LIBRARY_RELEASE})
- if (MSVC)
- set(BOOST_SHARED_SYSTEM_LIBRARY ${Boost_SYSTEM_LIBRARY_RELEASE})
- endif()
- endif()
-else()
- # Find static Boost libraries.
- set(Boost_USE_STATIC_LIBS ON)
- if (MSVC)
- find_package(Boost COMPONENTS regex system filesystem REQUIRED)
- else()
- find_package(Boost COMPONENTS regex REQUIRED)
- endif()
- if ("${UPPERCASE_BUILD_TYPE}" STREQUAL "DEBUG")
- set(BOOST_STATIC_REGEX_LIBRARY ${Boost_REGEX_LIBRARY_DEBUG})
- if (MSVC)
- set(BOOST_STATIC_SYSTEM_LIBRARY ${Boost_SYSTEM_LIBRARY_DEBUG})
- set(BOOST_STATIC_FILESYSTEM_LIBRARY ${Boost_FILESYSTEM_LIBRARY_DEBUG})
- endif()
- else()
- set(BOOST_STATIC_REGEX_LIBRARY ${Boost_REGEX_LIBRARY_RELEASE})
- if (MSVC)
- set(BOOST_STATIC_SYSTEM_LIBRARY ${Boost_SYSTEM_LIBRARY_RELEASE})
- set(BOOST_STATIC_FILESYSTEM_LIBRARY ${Boost_FILESYSTEM_LIBRARY_RELEASE})
- endif()
- endif()
-endif()
-
-message(STATUS "Boost include dir: " ${Boost_INCLUDE_DIRS})
-message(STATUS "Boost libraries: " ${Boost_LIBRARIES})
-if (PARQUET_BOOST_USE_SHARED)
- add_library(boost_shared_regex SHARED IMPORTED)
- if (MSVC)
- set_target_properties(boost_shared_regex
- PROPERTIES IMPORTED_IMPLIB "${BOOST_SHARED_REGEX_LIBRARY}")
- else()
- set_target_properties(boost_shared_regex
- PROPERTIES IMPORTED_LOCATION "${BOOST_SHARED_REGEX_LIBRARY}")
- endif()
- if (MSVC)
- add_library(boost_shared_system SHARED IMPORTED)
- set_target_properties(boost_shared_system
- PROPERTIES IMPORTED_IMPLIB "${BOOST_SHARED_SYSTEM_LIBRARY}")
- endif()
-else()
- add_library(boost_static_regex STATIC IMPORTED)
- set_target_properties(boost_static_regex PROPERTIES IMPORTED_LOCATION ${BOOST_STATIC_REGEX_LIBRARY})
- if (MSVC)
- add_library(boost_static_system STATIC IMPORTED)
- set_target_properties(boost_static_system PROPERTIES IMPORTED_LOCATION ${BOOST_STATIC_SYSTEM_LIBRARY})
- add_library(boost_static_filesystem STATIC IMPORTED)
- set_target_properties(boost_static_filesystem PROPERTIES IMPORTED_LOCATION ${BOOST_STATIC_FILESYSTEM_LIBRARY})
- endif()
-endif()
-
-include_directories(SYSTEM ${Boost_INCLUDE_DIRS})
-set(LIBS ${LIBS} ${Boost_LIBRARIES})
-
-# ----------------------------------------------------------------------
-# ZLIB
-
-# ----------------------------------------------------------------------
-# Thrift
-
-# find thrift headers and libs
-find_package(Thrift)
-
-if (NOT THRIFT_FOUND)
- set(ZLIB_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/zlib_ep/src/zlib_ep-install")
- set(ZLIB_HOME "${ZLIB_PREFIX}")
- set(ZLIB_INCLUDE_DIR "${ZLIB_PREFIX}/include")
- if (MSVC)
- if (${UPPERCASE_BUILD_TYPE} STREQUAL "DEBUG")
- set(ZLIB_STATIC_LIB_NAME zlibstaticd.lib)
- else()
- set(ZLIB_STATIC_LIB_NAME zlibstatic.lib)
- endif()
- else()
- set(ZLIB_STATIC_LIB_NAME libz.a)
- endif()
- set(ZLIB_STATIC_LIB "${ZLIB_PREFIX}/lib/${ZLIB_STATIC_LIB_NAME}")
- set(ZLIB_CMAKE_ARGS -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}
- -DCMAKE_INSTALL_PREFIX=${ZLIB_PREFIX}
- -DCMAKE_C_FLAGS=${EP_C_FLAGS}
- -DBUILD_SHARED_LIBS=OFF)
- ExternalProject_Add(zlib_ep
- URL "http://zlib.net/fossils/zlib-1.2.8.tar.gz"
- BUILD_BYPRODUCTS "${ZLIB_STATIC_LIB}"
- ${ZLIB_BUILD_BYPRODUCTS}
- ${EP_LOG_OPTIONS}
- CMAKE_ARGS ${ZLIB_CMAKE_ARGS})
-
- set(THRIFT_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/thrift_ep/src/thrift_ep-install")
- set(THRIFT_HOME "${THRIFT_PREFIX}")
- set(THRIFT_INCLUDE_DIR "${THRIFT_PREFIX}/include")
- set(THRIFT_COMPILER "${THRIFT_PREFIX}/bin/thrift")
- set(THRIFT_CMAKE_ARGS "-DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}"
- "-DCMAKE_CXX_FLAGS=${EP_CXX_FLAGS}"
- "-DCMAKE_C_FLAGS=${EP_C_FLAGS}"
- "-DCMAKE_INSTALL_PREFIX=${THRIFT_PREFIX}"
- "-DCMAKE_INSTALL_RPATH=${THRIFT_PREFIX}/lib"
- "-DBUILD_SHARED_LIBS=OFF"
- "-DBUILD_TESTING=OFF"
- "-DBUILD_EXAMPLES=OFF"
- "-DBUILD_TUTORIALS=OFF"
- "-DWITH_QT4=OFF"
- "-DWITH_C_GLIB=OFF"
- "-DWITH_JAVA=OFF"
- "-DWITH_PYTHON=OFF"
- "-DWITH_HASKELL=OFF"
- "-DWITH_CPP=ON"
- "-DWITH_STATIC_LIB=ON"
- "-DWITH_LIBEVENT=OFF"
- )
-
- # Thrift also uses boost. Forward important boost settings if there were ones passed.
- if (DEFINED BOOST_ROOT)
- set(THRIFT_CMAKE_ARGS ${THRIFT_CMAKE_ARGS} "-DBOOST_ROOT=${BOOST_ROOT}")
- endif()
- if (DEFINED Boost_NAMESPACE)
- set(THRIFT_CMAKE_ARGS ${THRIFT_CMAKE_ARGS} "-DBoost_NAMESPACE=${Boost_NAMESPACE}")
- endif()
-
- set(THRIFT_STATIC_LIB_NAME "${CMAKE_STATIC_LIBRARY_PREFIX}thrift")
- if (MSVC)
- if (PARQUET_USE_STATIC_CRT)
- set(THRIFT_STATIC_LIB_NAME "${THRIFT_STATIC_LIB_NAME}mt")
- set(THRIFT_CMAKE_ARGS ${THRIFT_CMAKE_ARGS} "-DWITH_MT=ON")
- else()
- set(THRIFT_STATIC_LIB_NAME "${THRIFT_STATIC_LIB_NAME}md")
- set(THRIFT_CMAKE_ARGS ${THRIFT_CMAKE_ARGS} "-DWITH_MT=OFF")
- endif()
- endif()
- if (${UPPERCASE_BUILD_TYPE} STREQUAL "DEBUG")
- set(THRIFT_STATIC_LIB_NAME "${THRIFT_STATIC_LIB_NAME}d")
- endif()
- set(THRIFT_STATIC_LIB "${THRIFT_PREFIX}/lib/${THRIFT_STATIC_LIB_NAME}${CMAKE_STATIC_LIBRARY_SUFFIX}")
-
- if (MSVC)
- set(WINFLEXBISON_VERSION 2.4.9)
- set(WINFLEXBISON_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/winflexbison_ep/src/winflexbison_ep-install")
- ExternalProject_Add(winflexbison_ep
- URL https://github.com/lexxmark/winflexbison/releases/download/v.${WINFLEXBISON_VERSION}/win_flex_bison-${WINFLEXBISON_VERSION}.zip
- URL_HASH MD5=a2e979ea9928fbf8567e995e9c0df765
- SOURCE_DIR ${WINFLEXBISON_PREFIX}
- CONFIGURE_COMMAND ""
- BUILD_COMMAND ""
- INSTALL_COMMAND ""
- ${EP_LOG_OPTIONS})
- set(THRIFT_DEPENDENCIES ${THRIFT_DEPENDENCIES} winflexbison_ep)
-
- set(THRIFT_CMAKE_ARGS "-DFLEX_EXECUTABLE=${WINFLEXBISON_PREFIX}/win_flex.exe"
- "-DBISON_EXECUTABLE=${WINFLEXBISON_PREFIX}/win_bison.exe"
- "-DZLIB_INCLUDE_DIR=${ZLIB_INCLUDE_DIR}"
- "-DZLIB_LIBRARY=${ZLIB_STATIC_LIB}"
- "-DWITH_SHARED_LIB=OFF"
- "-DWITH_PLUGIN=OFF"
- ${THRIFT_CMAKE_ARGS})
- set(THRIFT_DEPENDENCIES ${THRIFT_DEPENDENCIES} zlib_ep)
- elseif (APPLE)
- if (DEFINED BISON_EXECUTABLE)
- set(THRIFT_CMAKE_ARGS "-DBISON_EXECUTABLE=${BISON_EXECUTABLE}"
- ${THRIFT_CMAKE_ARGS})
- endif()
- endif()
-
- ExternalProject_Add(thrift_ep
- URL "http://archive.apache.org/dist/thrift/${THRIFT_VENDOR_VERSION}/thrift-${THRIFT_VENDOR_VERSION}.tar.gz"
- BUILD_BYPRODUCTS "${THRIFT_STATIC_LIB}" "${THRIFT_COMPILER}"
- CMAKE_ARGS ${THRIFT_CMAKE_ARGS}
- DEPENDS ${THRIFT_DEPENDENCIES}
- ${EP_LOG_OPTIONS})
-
- set(THRIFT_VENDORED 1)
-else()
- set(THRIFT_VENDORED 0)
-endif()
-
-include_directories(SYSTEM ${THRIFT_INCLUDE_DIR} ${THRIFT_INCLUDE_DIR}/thrift)
-message(STATUS "Thrift include dir: ${THRIFT_INCLUDE_DIR}")
-message(STATUS "Thrift static library: ${THRIFT_STATIC_LIB}")
-message(STATUS "Thrift compiler: ${THRIFT_COMPILER}")
-message(STATUS "Thrift version: ${THRIFT_VERSION}")
-add_library(thriftstatic STATIC IMPORTED)
-set_target_properties(thriftstatic PROPERTIES IMPORTED_LOCATION ${THRIFT_STATIC_LIB})
-
-if (THRIFT_VENDORED)
- add_dependencies(thriftstatic thrift_ep)
-endif()
-
-if (PARQUET_THRIFT_USE_BOOST)
- add_definitions(-DPARQUET_THRIFT_USE_BOOST)
- message(STATUS "Using Boost in Thrift header")
-endif()
-
-## GTest
-if(PARQUET_BUILD_TESTS AND NOT IGNORE_OPTIONAL_PACKAGES)
- add_custom_target(unittest ctest -L unittest)
-
- if("$ENV{GTEST_HOME}" STREQUAL "")
- if(APPLE)
- set(GTEST_CMAKE_CXX_FLAGS "${EP_CXX_FLAGS} -DGTEST_USE_OWN_TR1_TUPLE=1 -Wno-unused-value -Wno-ignored-attributes")
- elseif(MSVC)
- # Workaround https://github.com/google/googletest/issues/1111 until
- # a new gtest version is released
- set(GTEST_CMAKE_CXX_FLAGS "${EP_CXX_FLAGS} -D_SILENCE_TR1_NAMESPACE_DEPRECATION_WARNING")
- else()
- set(GTEST_CMAKE_CXX_FLAGS "${EP_CXX_FLAGS}")
- endif()
-
- set(GTEST_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/googletest_ep-prefix/src/googletest_ep")
- set(GTEST_INCLUDE_DIR "${GTEST_PREFIX}/include")
- set(GTEST_STATIC_LIB
- "${GTEST_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}gtest${CMAKE_STATIC_LIBRARY_SUFFIX}")
- set(GTEST_MAIN_STATIC_LIB
- "${GTEST_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}gtest_main${CMAKE_STATIC_LIBRARY_SUFFIX}")
-
- set(GTEST_CMAKE_ARGS -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}
- -DCMAKE_INSTALL_PREFIX=${GTEST_PREFIX}
- -DCMAKE_CXX_FLAGS=${GTEST_CMAKE_CXX_FLAGS})
-
- if (MSVC AND NOT PARQUET_USE_STATIC_CRT)
- set(GTEST_CMAKE_ARGS ${GTEST_CMAKE_ARGS} -Dgtest_force_shared_crt=ON)
- endif()
-
- ExternalProject_Add(googletest_ep
- URL "https://github.com/google/googletest/archive/release-${GTEST_VERSION}.tar.gz"
- BUILD_BYPRODUCTS "${GTEST_STATIC_LIB}" "${GTEST_MAIN_STATIC_LIB}"
- CMAKE_ARGS ${GTEST_CMAKE_ARGS}
- ${EP_LOG_OPTIONS})
- set(GTEST_VENDORED 1)
- else()
- find_package(GTest REQUIRED)
- set(GTEST_VENDORED 0)
- endif()
-
- message(STATUS "GTest include dir: ${GTEST_INCLUDE_DIR}")
- message(STATUS "GTest static library: ${GTEST_STATIC_LIB}")
- include_directories(SYSTEM ${GTEST_INCLUDE_DIR})
-
- add_library(gtest STATIC IMPORTED)
- set_target_properties(gtest PROPERTIES IMPORTED_LOCATION ${GTEST_STATIC_LIB})
-
- add_library(gtest_main STATIC IMPORTED)
- set_target_properties(gtest_main PROPERTIES IMPORTED_LOCATION
- ${GTEST_MAIN_STATIC_LIB})
-
- if(GTEST_VENDORED)
- add_dependencies(gtest googletest_ep)
- add_dependencies(gtest_main googletest_ep)
- endif()
-endif()
-
-## Google Benchmark
-if ("$ENV{GBENCHMARK_HOME}" STREQUAL "")
- set(GBENCHMARK_HOME ${THIRDPARTY_DIR}/installed)
-endif()
-
-if(PARQUET_BUILD_BENCHMARKS AND NOT IGNORE_OPTIONAL_PACKAGES)
- add_custom_target(runbenchmark ctest -L benchmark)
-
- if("$ENV{GBENCHMARK_HOME}" STREQUAL "")
- set(GBENCHMARK_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/gbenchmark_ep/src/gbenchmark_ep-install")
- set(GBENCHMARK_INCLUDE_DIR "${GBENCHMARK_PREFIX}/include")
- set(GBENCHMARK_STATIC_LIB "${GBENCHMARK_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}benchmark${CMAKE_STATIC_LIBRARY_SUFFIX}")
- set(GBENCHMARK_CMAKE_ARGS
- "-DCMAKE_INSTALL_PREFIX:PATH=${GBENCHMARK_PREFIX}"
- "-DBENCHMARK_ENABLE_TESTING=OFF"
- "-DCMAKE_CXX_FLAGS=${EP_CXX_FLAGS}")
- if (MSVC)
- set(GBENCHMARK_CMAKE_ARGS ${GBENCHMARK_CMAKE_ARGS} "-DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}")
- else()
- set(GBENCHMARK_CMAKE_ARGS ${GBENCHMARK_CMAKE_ARGS} "-DCMAKE_BUILD_TYPE=Release")
- endif()
- if (APPLE)
- set(GBENCHMARK_CMAKE_ARGS ${GBENCHMARK_CMAKE_ARGS} "-DBENCHMARK_USE_LIBCXX=ON")
- endif()
-
- ExternalProject_Add(gbenchmark_ep
- URL "https://github.com/google/benchmark/archive/v${GBENCHMARK_VERSION}.tar.gz"
- BUILD_BYPRODUCTS "${GBENCHMARK_STATIC_LIB}"
- CMAKE_ARGS ${GBENCHMARK_CMAKE_ARGS}
- ${EP_LOG_OPTIONS})
- set(GBENCHMARK_VENDORED 1)
- else()
- find_package(GBenchmark REQUIRED)
- set(GBENCHMARK_VENDORED 0)
- endif()
-
- message(STATUS "GBenchmark include dir: ${GBENCHMARK_INCLUDE_DIR}")
- message(STATUS "GBenchmark static library: ${GBENCHMARK_STATIC_LIB}")
- include_directories(SYSTEM ${GBENCHMARK_INCLUDE_DIR})
- add_library(gbenchmark STATIC IMPORTED)
- set_target_properties(gbenchmark PROPERTIES IMPORTED_LOCATION ${GBENCHMARK_STATIC_LIB})
-
- if(GBENCHMARK_VENDORED)
- add_dependencies(gbenchmark gbenchmark_ep)
- endif()
-endif()
-
-# ----------------------------------------------------------------------
-# Apache Arrow
-
-find_package(Arrow)
-if (NOT ARROW_FOUND)
- include(ArrowExternalProject)
- set(ARROW_VENDORED 1)
-else()
- set(ARROW_VENDORED 0)
-endif()
-
-include_directories(SYSTEM ${ARROW_INCLUDE_DIR})
-add_library(arrow SHARED IMPORTED)
-if(MSVC)
- set_target_properties(arrow
- PROPERTIES IMPORTED_IMPLIB "${ARROW_SHARED_IMPLIB}")
-else()
- set_target_properties(arrow
- PROPERTIES IMPORTED_LOCATION "${ARROW_SHARED_LIB}")
-endif()
-add_library(arrow_static STATIC IMPORTED)
-set_target_properties(arrow_static PROPERTIES IMPORTED_LOCATION ${ARROW_STATIC_LIB})
-
-if (ARROW_VENDORED)
- add_dependencies(arrow arrow_ep)
- add_dependencies(arrow_static arrow_ep)
-endif()
diff --git a/cmake_modules/clean-all.cmake b/cmake_modules/clean-all.cmake
deleted file mode 100644
index 38e61da..0000000
--- a/cmake_modules/clean-all.cmake
+++ /dev/null
@@ -1,31 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-set(cmake_generated ${CMAKE_BINARY_DIR}/CMakeCache.txt
- ${CMAKE_BINARY_DIR}/CTestTestfile.cmake
- ${CMAKE_BINARY_DIR}/cmake_install.cmake
- ${CMAKE_BINARY_DIR}/Makefile
- ${CMAKE_BINARY_DIR}/CMakeFiles
-)
-
-foreach(file ${cmake_generated})
-
- if (EXISTS ${file})
- file(REMOVE_RECURSE ${file})
- endif()
-
-endforeach(file)
diff --git a/cmake_modules/parquet-cppConfig.cmake.in b/cmake_modules/parquet-cppConfig.cmake.in
deleted file mode 100644
index 47021d0..0000000
--- a/cmake_modules/parquet-cppConfig.cmake.in
+++ /dev/null
@@ -1,32 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-set(PARQUET-CPP_VERSION x.y.z)
-
-@PACKAGE_INIT@
-
-set_and_check(PARQUET-CPP_INCLUDE_DIR "@PACKAGE_CMAKE_INSTALL_INCLUDEDIR@")
-set_and_check(PARQUET-CPP_LIB_DIR "@PACKAGE_CMAKE_INSTALL_LIBDIR@")
-
-set(_TMP_SUFFIX ${CMAKE_FIND_LIBRARY_SUFFIXES})
-FIND_LIBRARY(PARQUET_LIBRARY NAMES parquet PATHS ${PARQUET-CPP_LIB_DIR})
-set(PARQUET-CPP_LIBRARIES ${PARQUET_LIBRARY})
-
-set(CMAKE_FIND_LIBRARY_SUFFIXES ".a")
-FIND_LIBRARY(PARQUET_STATIC_LIBRARY NAMES parquet PATHS ${PARQUET-CPP_LIB_DIR})
-set(PARQUET-CPP_STATIC_LIBRARIES ${PARQUET_STATIC_LIBRARY})
-set(CMAKE_FIND_LIBRARY_SUFFIXES ${_TMP_SUFFIX})
diff --git a/data/alltypes_dictionary.parquet b/data/alltypes_dictionary.parquet
deleted file mode 100755
index e6da6ab..0000000
--- a/data/alltypes_dictionary.parquet
+++ /dev/null
Binary files differ
diff --git a/data/alltypes_plain.parquet b/data/alltypes_plain.parquet
deleted file mode 100755
index a63f5dc..0000000
--- a/data/alltypes_plain.parquet
+++ /dev/null
Binary files differ
diff --git a/data/alltypes_plain.snappy.parquet b/data/alltypes_plain.snappy.parquet
deleted file mode 100755
index 9809d67..0000000
--- a/data/alltypes_plain.snappy.parquet
+++ /dev/null
Binary files differ
diff --git a/data/bloom_filter.bin b/data/bloom_filter.bin
deleted file mode 100644
index c0e30ce..0000000
--- a/data/bloom_filter.bin
+++ /dev/null
Binary files differ
diff --git a/data/fixed_length_decimal.parquet b/data/fixed_length_decimal.parquet
deleted file mode 100644
index 69fce53..0000000
--- a/data/fixed_length_decimal.parquet
+++ /dev/null
Binary files differ
diff --git a/data/fixed_length_decimal_legacy.parquet b/data/fixed_length_decimal_legacy.parquet
deleted file mode 100644
index b0df62a..0000000
--- a/data/fixed_length_decimal_legacy.parquet
+++ /dev/null
Binary files differ
diff --git a/data/int32_decimal.parquet b/data/int32_decimal.parquet
deleted file mode 100644
index 5bf2d4e..0000000
--- a/data/int32_decimal.parquet
+++ /dev/null
Binary files differ
diff --git a/data/int64_decimal.parquet b/data/int64_decimal.parquet
deleted file mode 100644
index 5043bca..0000000
--- a/data/int64_decimal.parquet
+++ /dev/null
Binary files differ
diff --git a/data/nation.dict-malformed.parquet b/data/nation.dict-malformed.parquet
deleted file mode 100644
index 5008ac0..0000000
--- a/data/nation.dict-malformed.parquet
+++ /dev/null
Binary files differ
diff --git a/dev/README.md b/dev/README.md
deleted file mode 100644
index d567672..0000000
--- a/dev/README.md
+++ /dev/null
@@ -1,88 +0,0 @@
-<!---
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License. See accompanying LICENSE file.
--->
-
-# Parquet Developer Scripts
-This directory contains scripts useful to developers when packaging,
-testing, or committing to Parquet.
-
-Merging a pull request requires being a committer on the project.
-
-* How to merge a Pull request:
-have an apache and apache-github remote setup
-```
-git remote add apache-github https://github.com/apache/parquet-cpp.git
-git remote add apache git@github.com:apache/parquet-cpp.git
-```
-run the following command
-```
-dev/merge_parquet_pr.py
-```
-
-Note:
-* The parent directory of your parquet repository must be called parquet-cpp
-* Without jira-python installed you'll have to close the JIRA manually
-
-example output:
-```
-Which pull request would you like to merge? (e.g. 34):
-```
-Type the pull request number (from https://github.com/apache/parquet-cpp/pulls) and hit enter.
-```
-=== Pull Request #X ===
-title Blah Blah Blah
-source repo/branch
-target master
-url https://api.github.com/repos/apache/parquet-cpp/pulls/X
-
-Proceed with merging pull request #3? (y/n):
-```
-If this looks good, type y and hit enter.
-```
-From git-wip-us.apache.org:/repos/asf/parquet-cpp.git
- * [new branch] master -> PR_TOOL_MERGE_PR_3_MASTER
-Switched to branch 'PR_TOOL_MERGE_PR_3_MASTER'
-
-Merge complete (local ref PR_TOOL_MERGE_PR_3_MASTER). Push to apache? (y/n):
-```
-A local branch with the merge has been created.
-type y and hit enter to push it to apache master
-```
-Counting objects: 67, done.
-Delta compression using up to 4 threads.
-Compressing objects: 100% (26/26), done.
-Writing objects: 100% (36/36), 5.32 KiB, done.
-Total 36 (delta 17), reused 0 (delta 0)
-To git-wip-us.apache.org:/repos/asf/parquet-cpp.git
- b767ac4..485658a PR_TOOL_MERGE_PR_X_MASTER -> master
-Restoring head pointer to b767ac4e
-Note: checking out 'b767ac4e'.
-
-You are in 'detached HEAD' state. You can look around, make experimental
-changes and commit them, and you can discard any commits you make in this
-state without impacting any branches by performing another checkout.
-
-If you want to create a new branch to retain commits you create, you may
-do so (now or later) by using -b with the checkout command again. Example:
-
- git checkout -b new_branch_name
-
-HEAD is now at b767ac4... Update README.md
-Deleting local branch PR_TOOL_MERGE_PR_X
-Deleting local branch PR_TOOL_MERGE_PR_X_MASTER
-Pull request #X merged!
-Merge hash: 485658a5
-
-Would you like to pick 485658a5 into another branch? (y/n):
-```
-For now just say n as we have 1 branch
diff --git a/dev/committers-guide.md b/dev/committers-guide.md
deleted file mode 100644
index f3fddfd..0000000
--- a/dev/committers-guide.md
+++ /dev/null
@@ -1,118 +0,0 @@
-<!---
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License. See accompanying LICENSE file.
-
- Original imported from Apache Aurora.
--->
-
-Committer's Guide
-=================
-
-Information for official Apache Parquet committers.
-
-Setting up your email account
------------------------------
-Once your Apache ID has been set up you can configure your account and add ssh keys and setup an
-email forwarding address at
-
- http://id.apache.org
-
-Additional instructions for setting up your new committer email can be found at
-
- http://www.apache.org/dev/user-email.html
-
-The recommended setup is to configure all services (mailing lists, JIRA) to send
-emails to your @apache.org email address.
-
-
-Creating a gpg key for releases
--------------------------------
-In order to create a release candidate you will need a gpg key published to an external key server
-and that key will need to be added to our KEYS file as well.
-
-1. Create a key:
-
- gpg --gen-key
-
-2. Add your gpg key to the Apache Parquet {format,mr,cpp} KEYS file:
-
- git clone git@github.com:apache/parquet-cpp.git
- (gpg --list-sigs <KEY ID> && gpg --armor --export <KEY ID>) >> KEYS
- git add KEYS && git commit -m "Adding gpg key for <APACHE ID>"
- ./rbt post -o -g
-
-3. Publish the key to an external key server:
-
- gpg --keyserver pgp.mit.edu --send-keys <KEY ID>
-
-4. Update the changes to the KEYS file to the Apache Parquet svn dist locations listed below:
-
- https://dist.apache.org/repos/dist/dev/parquet/KEYS
- https://dist.apache.org/repos/dist/release/parquet/KEYS
-
-5. Add your key to git config for use with the release scripts:
-
- git config --global user.signingkey <KEY ID>
-
-
-Creating a release
-------------------
-The following will guide you through the steps to create a release candidate, vote, and finally an
-official Apache Parquet C++ release. Before starting your gpg key should be in the KEYS file and you
-must have access to commit to the dist.a.o repositories.
-
-1. Ensure that all issues resolved for this release candidate are tagged with the correct Fix
-Version in JIRA, the changelog script will use this to generate the CHANGELOG in step #2.
-To assign the fix version:
-
- * Look up the [previous release date](https://issues.apache.org/jira/browse/parquet/?selectedTab=com.atlassian.jira.jira-projects-plugin:versions-panel).
- * Query all issues resolved after that release date: `project = PARQUET AND status in (resolved, Closed) and fixVersion is empty and resolutiondate >= "YYYY/MM/DD"`
- * In the upper right corner of the query result, select Tools > Bulk Edit.
- * Select all issues > edit issue > set 'Change Fix Version/s' to the release version.
- * Make sure to uncheck 'Send mail for this update' at the bottom.
-
-2. Prepare RELEASE-NOTES.md for the release. This just boils down to removing the "(Not yet
-released)" suffix from the impending release.
-
-2. Create a release candidate. This will automatically update the CHANGELOG and commit it, create a
-branch and update the current version within the trunk. To create a minor version update and publish
-it run
-
- ./dev/release/release-candidate -l m -p
-
-3. Update, if necessary, the draft email created from the `release-candidate` script in step #2 and
-send the [VOTE] email to the dev@ mailing list. You can verify the release signature and checksums
-by running
-
- ./dev/release/verify-release-candidate
-
-4. Wait for the vote to complete. If the vote fails close the vote by replying to the initial [VOTE]
-email sent in step #3 by editing the subject to [RESULT][VOTE] ... and noting the failure reason
-(example [here](http://markmail.org/message/d4d6xtvj7vgwi76f)). You'll also need to manually revert
-the commits generated by the release candidate script that incremented the snapshot version and
-updated the changelog. Once that is done, now address any issues and go back to step #1 and run
-again, this time you will use the -r flag to increment the release candidate version. This will
-automatically clean up the release candidate rc0 branch and source distribution.
-
- ./dev/release/release-candidate -l m -r 1 -p
-
-5. Once the vote has successfully passed create the release
-
-**IMPORTANT: make sure to use the correct release at this final step (e.g.: `-r 1` if rc1 candidate
-has been voted for). Once the release tag is pushed it will be very hard to undo due to remote
-git pre-receive hook explicitly forbidding release tag manipulations.**
-
- ./dev/release/release
-
-6. Update the draft email created fom the `release` script in step #5 to include the Apache ID's for
-all binding votes and send the [RESULT][VOTE] email to the dev@ mailing list.
-
diff --git a/dev/merge_parquet_pr.py b/dev/merge_parquet_pr.py
deleted file mode 100755
index dcb7af6..0000000
--- a/dev/merge_parquet_pr.py
+++ /dev/null
@@ -1,354 +0,0 @@
-#!/usr/bin/env python
-
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# Utility for creating well-formed pull request merges and pushing them to Apache.
-# usage: ./apache-pr-merge.py (see config env vars below)
-#
-# This utility assumes you already have local a Parquet git folder and that you
-# have added remotes corresponding to both (i) the github apache Parquet
-# mirror and (ii) the apache git repo.
-
-import json
-import os
-import re
-import subprocess
-import sys
-import tempfile
-import urllib2
-import getpass
-
-try:
- import jira.client
- JIRA_IMPORTED = True
-except ImportError:
- JIRA_IMPORTED = False
-
-# Location of your Parquet git development area
-PARQUET_HOME = os.path.abspath(__file__).rsplit("/", 2)[0]
-PROJECT_NAME = PARQUET_HOME.rsplit("/", 1)[1]
-print "PARQUET_HOME = " + PARQUET_HOME
-print "PROJECT_NAME = " + PROJECT_NAME
-
-# Remote name which points to the Gihub site
-PR_REMOTE_NAME = os.environ.get("PR_REMOTE_NAME", "apache-github")
-# Remote name which points to Apache git
-PUSH_REMOTE_NAME = os.environ.get("PUSH_REMOTE_NAME", "apache")
-# ASF JIRA username
-JIRA_USERNAME = os.environ.get("JIRA_USERNAME")
-# ASF JIRA password
-JIRA_PASSWORD = os.environ.get("JIRA_PASSWORD")
-
-GITHUB_BASE = "https://github.com/apache/" + PROJECT_NAME + "/pull"
-GITHUB_API_BASE = "https://api.github.com/repos/apache/" + PROJECT_NAME
-JIRA_BASE = "https://issues.apache.org/jira/browse"
-JIRA_API_BASE = "https://issues.apache.org/jira"
-# Prefix added to temporary branches
-BRANCH_PREFIX = "PR_TOOL"
-
-os.chdir(PARQUET_HOME)
-
-
-def get_json(url):
- try:
- return json.load(urllib2.urlopen(url))
- except urllib2.HTTPError as e:
- print "Unable to fetch URL, exiting: %s" % url
- sys.exit(-1)
-
-
-def fail(msg):
- print msg
- clean_up()
- sys.exit(-1)
-
-
-def run_cmd(cmd):
- if isinstance(cmd, list):
- return subprocess.check_output(cmd)
- else:
- return subprocess.check_output(cmd.split(" "))
-
-
-def continue_maybe(prompt):
- result = raw_input("\n%s (y/n): " % prompt)
- if result.lower() != "y":
- fail("Okay, exiting")
-
-
-original_head = run_cmd("git rev-parse HEAD")[:8]
-
-
-def clean_up():
- print "Restoring head pointer to %s" % original_head
- run_cmd("git checkout %s" % original_head)
-
- branches = run_cmd("git branch").replace(" ", "").split("\n")
-
- for branch in filter(lambda x: x.startswith(BRANCH_PREFIX), branches):
- print "Deleting local branch %s" % branch
- run_cmd("git branch -D %s" % branch)
-
-
-# merge the requested PR and return the merge hash
-def merge_pr(pr_num, target_ref):
- pr_branch_name = "%s_MERGE_PR_%s" % (BRANCH_PREFIX, pr_num)
- target_branch_name = "%s_MERGE_PR_%s_%s" % (BRANCH_PREFIX, pr_num, target_ref.upper())
- run_cmd("git fetch %s pull/%s/head:%s" % (PR_REMOTE_NAME, pr_num, pr_branch_name))
- run_cmd("git fetch %s %s:%s" % (PUSH_REMOTE_NAME, target_ref, target_branch_name))
- run_cmd("git checkout %s" % target_branch_name)
-
- had_conflicts = False
- try:
- run_cmd(['git', 'merge', pr_branch_name, '--squash'])
- except Exception as e:
- msg = "Error merging: %s\nWould you like to manually fix-up this merge?" % e
- continue_maybe(msg)
- msg = "Okay, please fix any conflicts and 'git add' conflicting files... Finished?"
- continue_maybe(msg)
- had_conflicts = True
-
- commit_authors = run_cmd(['git', 'log', 'HEAD..%s' % pr_branch_name,
- '--pretty=format:%an <%ae>']).split("\n")
- distinct_authors = sorted(set(commit_authors),
- key=lambda x: commit_authors.count(x), reverse=True)
- primary_author = distinct_authors[0]
- commits = run_cmd(['git', 'log', 'HEAD..%s' % pr_branch_name,
- '--pretty=format:%h [%an] %s']).split("\n\n")
-
- merge_message_flags = []
-
- merge_message_flags += ["-m", title]
- if body != None:
- merge_message_flags += ["-m", body]
-
- authors = "\n".join(["Author: %s" % a for a in distinct_authors])
-
- merge_message_flags += ["-m", authors]
-
- if had_conflicts:
- committer_name = run_cmd("git config --get user.name").strip()
- committer_email = run_cmd("git config --get user.email").strip()
- message = "This patch had conflicts when merged, resolved by\nCommitter: %s <%s>" % (
- committer_name, committer_email)
- merge_message_flags += ["-m", message]
-
- # The string "Closes #%s" string is required for GitHub to correctly close the PR
- merge_message_flags += [
- "-m",
- "Closes #%s from %s and squashes the following commits:" % (pr_num, pr_repo_desc)]
- for c in commits:
- merge_message_flags += ["-m", c]
-
- run_cmd(['git', 'commit', '--author="%s"' % primary_author] + merge_message_flags)
-
- continue_maybe("Merge complete (local ref %s). Push to %s?" % (
- target_branch_name, PUSH_REMOTE_NAME))
-
- try:
- run_cmd('git push %s %s:%s' % (PUSH_REMOTE_NAME, target_branch_name, target_ref))
- except Exception as e:
- clean_up()
- fail("Exception while pushing: %s" % e)
-
- merge_hash = run_cmd("git rev-parse %s" % target_branch_name)[:8]
- clean_up()
- print("Pull request #%s merged!" % pr_num)
- print("Merge hash: %s" % merge_hash)
- return merge_hash
-
-
-def cherry_pick(pr_num, merge_hash, default_branch):
- pick_ref = raw_input("Enter a branch name [%s]: " % default_branch)
- if pick_ref == "":
- pick_ref = default_branch
-
- pick_branch_name = "%s_PICK_PR_%s_%s" % (BRANCH_PREFIX, pr_num, pick_ref.upper())
-
- run_cmd("git fetch %s %s:%s" % (PUSH_REMOTE_NAME, pick_ref, pick_branch_name))
- run_cmd("git checkout %s" % pick_branch_name)
- run_cmd("git cherry-pick -sx %s" % merge_hash)
-
- continue_maybe("Pick complete (local ref %s). Push to %s?" % (
- pick_branch_name, PUSH_REMOTE_NAME))
-
- try:
- run_cmd('git push %s %s:%s' % (PUSH_REMOTE_NAME, pick_branch_name, pick_ref))
- except Exception as e:
- clean_up()
- fail("Exception while pushing: %s" % e)
-
- pick_hash = run_cmd("git rev-parse %s" % pick_branch_name)[:8]
- clean_up()
-
- print("Pull request #%s picked into %s!" % (pr_num, pick_ref))
- print("Pick hash: %s" % pick_hash)
- return pick_ref
-
-
-def fix_version_from_branch(branch, versions):
- # Note: Assumes this is a sorted (newest->oldest) list of un-released versions
- if branch == "master":
- return versions[0]
- else:
- branch_ver = branch.replace("branch-", "")
- return filter(lambda x: x.name.startswith(branch_ver), versions)[-1]
-
-def exctract_jira_id(title):
- m = re.search('^(PARQUET-[0-9]+):.*$', title)
- if m and m.groups > 0:
- return m.group(1)
- else:
- fail("PR title should be prefixed by a jira id \"PARQUET-XXX: ...\", found: \"%s\"" % title)
-
-def check_jira(title):
- jira_id = exctract_jira_id(title)
- asf_jira = jira.client.JIRA({'server': JIRA_API_BASE},
- basic_auth=(JIRA_USERNAME, JIRA_PASSWORD))
- try:
- issue = asf_jira.issue(jira_id)
- except Exception as e:
- fail("ASF JIRA could not find %s\n%s" % (jira_id, e))
-
-def resolve_jira(title, merge_branches, comment):
- asf_jira = jira.client.JIRA({'server': JIRA_API_BASE},
- basic_auth=(JIRA_USERNAME, JIRA_PASSWORD))
-
- default_jira_id = exctract_jira_id(title)
-
- jira_id = raw_input("Enter a JIRA id [%s]: " % default_jira_id)
- if jira_id == "":
- jira_id = default_jira_id
-
- try:
- issue = asf_jira.issue(jira_id)
- except Exception as e:
- fail("ASF JIRA could not find %s\n%s" % (jira_id, e))
-
- cur_status = issue.fields.status.name
- cur_summary = issue.fields.summary
- cur_assignee = issue.fields.assignee
- if cur_assignee is None:
- cur_assignee = "NOT ASSIGNED!!!"
- else:
- cur_assignee = cur_assignee.displayName
-
- if cur_status == "Resolved" or cur_status == "Closed":
- fail("JIRA issue %s already has status '%s'" % (jira_id, cur_status))
- print ("=== JIRA %s ===" % jira_id)
- print ("summary\t\t%s\nassignee\t%s\nstatus\t\t%s\nurl\t\t%s/%s\n" % (
- cur_summary, cur_assignee, cur_status, JIRA_BASE, jira_id))
-
- versions = asf_jira.project_versions("PARQUET")
- versions = sorted(versions, key=lambda x: x.name, reverse=True)
- versions = filter(lambda x: x.raw['released'] is False, versions)
-
- default_fix_versions = map(lambda x: fix_version_from_branch(x, versions).name, merge_branches)
- for v in default_fix_versions:
- # Handles the case where we have forked a release branch but not yet made the release.
- # In this case, if the PR is committed to the master branch and the release branch, we
- # only consider the release branch to be the fix version. E.g. it is not valid to have
- # both 1.1.0 and 1.0.0 as fix versions.
- (major, minor, patch) = v.split(".")
- if patch == "0":
- previous = "%s.%s.%s" % (major, int(minor) - 1, 0)
- if previous in default_fix_versions:
- default_fix_versions = filter(lambda x: x != v, default_fix_versions)
- default_fix_versions = ",".join(default_fix_versions)
-
- fix_versions = raw_input("Enter comma-separated fix version(s) [%s]: " % default_fix_versions)
- if fix_versions == "":
- fix_versions = default_fix_versions
- fix_versions = fix_versions.replace(" ", "").split(",")
-
- def get_version_json(version_str):
- return filter(lambda v: v.name == version_str, versions)[0].raw
-
- jira_fix_versions = map(lambda v: get_version_json(v), fix_versions)
-
- resolve = filter(lambda a: a['name'] == "Resolve Issue", asf_jira.transitions(jira_id))[0]
- asf_jira.transition_issue(
- jira_id, resolve["id"], fixVersions=jira_fix_versions, comment=comment)
-
- print "Succesfully resolved %s with fixVersions=%s!" % (jira_id, fix_versions)
-
-
-if not JIRA_USERNAME:
- JIRA_USERNAME = raw_input("Env JIRA_USERNAME not set, please enter your JIRA username:")
-
-if not JIRA_PASSWORD:
- JIRA_PASSWORD = getpass.getpass("Env JIRA_PASSWORD not set, please enter your JIRA password:")
-
-branches = get_json("%s/branches" % GITHUB_API_BASE)
-branch_names = filter(lambda x: x.startswith("branch-"), [x['name'] for x in branches])
-# Assumes branch names can be sorted lexicographically
-# Julien: I commented this out as we don't have any "branch-*" branch yet
-#latest_branch = sorted(branch_names, reverse=True)[0]
-
-pr_num = raw_input("Which pull request would you like to merge? (e.g. 34): ")
-pr = get_json("%s/pulls/%s" % (GITHUB_API_BASE, pr_num))
-
-url = pr["url"]
-title = pr["title"]
-check_jira(title)
-body = pr["body"]
-target_ref = pr["base"]["ref"]
-user_login = pr["user"]["login"]
-base_ref = pr["head"]["ref"]
-pr_repo_desc = "%s/%s" % (user_login, base_ref)
-
-if pr["merged"] is True:
- print "Pull request %s has already been merged, assuming you want to backport" % pr_num
- merge_commit_desc = run_cmd([
- 'git', 'log', '--merges', '--first-parent',
- '--grep=pull request #%s' % pr_num, '--oneline']).split("\n")[0]
- if merge_commit_desc == "":
- fail("Couldn't find any merge commit for #%s, you may need to update HEAD." % pr_num)
-
- merge_hash = merge_commit_desc[:7]
- message = merge_commit_desc[8:]
-
- print "Found: %s" % message
- maybe_cherry_pick(pr_num, merge_hash, latest_branch)
- sys.exit(0)
-
-if not bool(pr["mergeable"]):
- msg = "Pull request %s is not mergeable in its current form.\n" % pr_num + \
- "Continue? (experts only!)"
- continue_maybe(msg)
-
-print ("\n=== Pull Request #%s ===" % pr_num)
-print ("title\t%s\nsource\t%s\ntarget\t%s\nurl\t%s" % (
- title, pr_repo_desc, target_ref, url))
-continue_maybe("Proceed with merging pull request #%s?" % pr_num)
-
-merged_refs = [target_ref]
-
-merge_hash = merge_pr(pr_num, target_ref)
-
-pick_prompt = "Would you like to pick %s into another branch?" % merge_hash
-while raw_input("\n%s (y/n): " % pick_prompt).lower() == "y":
- merged_refs = merged_refs + [cherry_pick(pr_num, merge_hash, latest_branch)]
-
-if JIRA_IMPORTED:
- continue_maybe("Would you like to update the associated JIRA?")
- jira_comment = "Issue resolved by pull request %s\n[%s/%s]" % (pr_num, GITHUB_BASE, pr_num)
- resolve_jira(title, merged_refs, jira_comment)
-else:
- print "Could not find jira-python library. Run 'sudo pip install jira-python' to install."
- print "Exiting without trying to close the associated JIRA."
diff --git a/dev/release/README.md b/dev/release/README.md
deleted file mode 100644
index 142ab64..0000000
--- a/dev/release/README.md
+++ /dev/null
@@ -1,30 +0,0 @@
-<!---
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License. See accompanying LICENSE file.
--->
-
-
-How to release
-==============
-
-1. Do a fresh `git clone` of the repository using the ASF git as origin.
-2. Set the git author to your gpg signing key using `git config --local user.email "<mail>"`.
-3. Create an RC using `./dev/release/release-candidate -r <rcNum> -v <version>` starting with `rcNum=0`.
- You could first do a dry-run by adding `-p` at the end.
-4. Check that the uploaded RC is valid using `./dev/release/verify-release-candidate <version> <rcNum>`.
-5. Send the generated mail to dev@parquet.apache.org.
-6. Wait for the vote to end, either proceed to the next step or abort the RC and continue with the
- necessary fixes.
-7. In the release SVN, copy the contents of the RC folder into the release folder.
-8. Delete the previous `parquet-cpp` release from SVN.
-9. Cherry-pick the commits in the `master-after-X` branch into master.
-10. Send out the release mail to `dev@parquet.apache.org` and `announce@apache.org`.
diff --git a/dev/release/changelog b/dev/release/changelog
deleted file mode 100755
index d6ed9f2..0000000
--- a/dev/release/changelog
+++ /dev/null
@@ -1,155 +0,0 @@
-#!/usr/bin/env ruby
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-# Updates the CHANGELOG for the specified Apache Parquet C++ release, if no
-# version override is specified then .parquetcppversion will be read and used.
-# The version string will need to match the Apache Parquet C++ jira tickets
-# fixVersion to generate this list.
-#
-require 'rubygems'
-require 'json'
-require 'uri'
-require 'net/http'
-require 'net/https'
-require 'openssl'
-require 'tempfile'
-
-def http_post(uri, params={}, headers={}, body=nil, debug=false)
- http = Net::HTTP.new(uri.host, uri.port)
- http.set_debug_output($stdout) if debug
- if uri.scheme == "https" || uri.port == 443
- http.use_ssl = true
- http.verify_mode = OpenSSL::SSL::VERIFY_NONE
- end
- response = nil
-
- begin
- request = Net::HTTP::Post.new(uri.request_uri)
- request.set_form_data(params) if (params && params.size > 1)
- request.basic_auth uri.user, uri.password if uri.user
- request.body = body if body
-
- headers.each_pair do |name, value|
- request[name] = value
- end
-
- response = http.request(request)
- rescue Exception => e
- raise "Error posting to #{uri.to_s}. #{e}"
- end
-
- response
-end
-
-def get_jira_issues_for_query(url, jql, startAt=0, maxResults=100)
- puts(jql)
- begin
- request_body = { :jql => jql, :startAt => startAt, :maxResults => maxResults}
- response = http_post(
- URI(url),
- {},
- {"content-type" => 'application/json', "accept" => "application/json"},
- request_body.to_json
- )
- rescue StandardError => e
- raise "Error executing jql query: #{jql}. #{e}"
- end
-
- begin
- results = JSON.parse(response.body)
- rescue StandardError => e
- raise "Error parsing json result"
- end
-
- results
-end
-
-def get_all_jira_issues_for_query(url, jql)
- startAt = 0
- maxResults = 100
-
- results = []
- begin
- res = get_jira_issues_for_query(url, jql, startAt, maxResults)
- results.concat(res['issues']) if res && res.has_key?('issues')
- startAt = startAt + maxResults
- end until (results.size() == res['total'] )
-
- results
-end
-
-base_dir = `git rev-parse --show-toplevel`.strip
-
-# Get the current version from the .parquetcppversion file if no version override is provided
-version = nil
-if ARGV[0].nil?
- version = nil
- Dir.chdir(base_dir) do
- version = File.read('.parquetcppversion').strip
- end
-else
- version = ARGV[0]
-end
-
-raise "Unable to read .parquetcppversion" if version.nil?
-
-jira_base_url = "https://issues.apache.org/jira"
-jira_search_url = "/rest/api/2/search"
-jira_url = jira_base_url + jira_search_url
-
-jql="project=PARQUET " \
- "AND fixVersion='cpp-#{version}' " \
- "AND status = Resolved " \
- "AND resolution in (Fixed, Done) " \
- "ORDER BY issuetype DESC"
-
-# Fetch all the issues available for the given jql query
-results = get_all_jira_issues_for_query(jira_url, jql)
-
-changelog = {}
-# Loop through and add all results
-results.each do |issue|
- key = issue['key']
- summary = issue['fields']['summary']
- type = issue['fields']['issuetype']['name']
-
- changelog_entry = "[#{key}] - #{summary}"
-
- if !changelog[type].nil?
- changelog[type] << changelog_entry
- else
- changelog[type] = [changelog_entry]
- end
-end
-
-# Merge the new updates and the existing changelog
-tmpfile = Tempfile.open('parquet-cpp.changelog')
-begin
- tmpfile.puts "Parquet C++ #{version}", "-" * 80
- changelog.keys.sort.each do |type|
- tmpfile.puts "## #{type}"
- changelog[type].each { |entry| tmpfile.puts " * #{entry}" }
- tmpfile.puts ""
- end
- # Append all the existing CHANGELOG entries and write the new CHANGELOG file
- tmpfile.puts ""
- changelog_file = File.join(base_dir, 'CHANGELOG')
- tmpfile.write File.read(changelog_file) if File.exist?(changelog_file)
- tmpfile.rewind
- tmpfile.flush
- File.open(changelog_file,"w+") {|f| f.write(tmpfile.read) }
-ensure
- tmpfile.close
- tmpfile.unlink
-end
diff --git a/dev/release/release-candidate b/dev/release/release-candidate
deleted file mode 100755
index 88aa27a..0000000
--- a/dev/release/release-candidate
+++ /dev/null
@@ -1,316 +0,0 @@
-#!/bin/bash
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-#
-# This script is used to create a release candidate. It will update the current
-# .parquetcppversion as well as creates a tag for the new release candidate and
-# publishes the source distribution and signatures to be voted on.
-#
-# master~1 (0.5.0-snapshot) ----- master (0.6.0-snapshot)
-# \---- 0.5.0 (0.5.0)
-#
-# A email template will be generated after successfully generating a release
-# candidate which will need to be sent to the dev@ and private@ mailing lists.
-#
-set -o errexit
-set -o nounset
-
-rc_number=0
-override_version=''
-parquetcpp_git_web_url='https://gitbox.apache.org/repos/asf?p=parquet-cpp.git'
-parquet_svn_dist_url='https://dist.apache.org/repos/dist/dev/parquet'
-
-function print_help_and_exit {
-cat <<EOF
-Apache Parquet C++ release candidate tool.
-
-Usage: $0 [-h] [-l p|m|M] [-r #] [-p | publish]
-
- -h Print this help message and exit
- -l Increment level, must be one of:
- p, patch (default)
- m, minor
- M, major
- -v Override the existing version in .parquetcppversion
- -r Release candidate number (default: 0)
- -p Publish the release candidate (default: dry-run, does not publish anything)
-EOF
-exit 0
-}
-
-publish=0
-increment_level="patch"
-rc_number=0
-while getopts ":hl:v:r:p" opt; do
- case $opt in
- l)
- case ${OPTARG} in
- 'p' | 'patch') increment_level='patch' ;;
- 'm' | 'minor') increment_level='minor' ;;
- 'M' | 'major') increment_level='major' ;;
- *) echo 'Unknown increment level'; exit 1 ;;
- esac
- ;;
- r)
- rc_number=${OPTARG}
- ;;
- p)
- publish=1
- ;;
- h)
- print_help_and_exit
- ;;
- v)
- override_version=${OPTARG}
- ;;
- * )
- echo "Unknown option: -$OPTARG"
- print_help_and_exit
- ;;
- esac
-done
-
-shift $(($OPTIND - 1))
-if [[ "${1:-dry-run}" == "publish" ]]; then
- publish=1
-fi
-
-# Update local repository
-git fetch --all -q
-git fetch --tags -q
-
-# Verify that this is a clean repository
-if [[ -n "`git status --porcelain`" ]]; then
- echo "ERROR: Please run from a clean git repository."
- exit 1
-elif [[ "`git rev-parse --abbrev-ref HEAD`" != "master" ]]; then
- echo "ERROR: This script must be run from master."
- exit 1
-fi
-
-if [[ ! -f .parquetcppversion ]]; then
- echo "Warrning: This script must be run from the root of the repository"
- exit 1
-fi
-
-# Calculate the new version string
-current_version=$(cat .parquetcppversion | tr '[a-z]' '[A-Z]')
-if ! [[ $current_version =~ .*-SNAPSHOT ]]; then
- echo "ERROR: .parquetcppversion is required to contain 'SNAPSHOT', it is ${current_version}"
- exit 1
-else
- if [[ $override_version != "" ]]; then
- current_version=$override_version
- fi
-
- major=`echo $current_version | cut -d. -f1`
- minor=`echo $current_version | cut -d. -f2`
- patch=`echo $current_version | cut -d. -f3 | cut -d- -f1`
-
- current_version="${major}.${minor}.${patch}"
-
- if [[ $increment_level == "patch" ]]; then
- new_master_version="${major}.${minor}.$((patch + 1))"
- elif [[ $increment_level == "minor" ]]; then
- new_master_version="${major}.$((minor + 1)).0"
- elif [[ $increment_level == "major" ]]; then
- new_master_version="$((major + 1)).0.0"
- else
- echo "Unknown release increment ${increment_level}"
- exit 1
- fi
-
- new_snapshot_version="${new_master_version}-SNAPSHOT"
-fi
-
-# Add the rc tag to the current version
-rc_version="${current_version}-rc${rc_number}"
-version_tag="apache-parquet-cpp-${current_version}"
-rc_version_tag="apache-parquet-cpp-${rc_version}"
-
-echo
-echo "Generating release candidate ${rc_version_tag}"
-echo
-
-# Make sure the tag does not exist
-if git rev-parse ${version_tag} >/dev/null 2>&1; then
- echo "ERROR: tag ${version_tag} exists."
- exit 1
-fi
-
-if git rev-parse ${rc_version_tag} >/dev/null 2>&1; then
- echo "ERROR: tag ${rc_version_tag} exists."
- exit 1
-fi
-
-# Reset instructions
-current_git_rev=$(git rev-parse HEAD)
-function print_reset_instructions {
-cat <<EOF
-To roll back your local repo you will need to run:
-
- git checkout master
- git reset --hard ${current_git_rev}
- git tag -d ${rc_version_tag}
- git branch -D stage_${rc_version}
-EOF
-}
-
-# If anything goes wrong from here then print roll back instructions before exiting.
-function print_rollback_instructions {
- echo "ERROR: Looks like something has failed while creating the release candidate."
- print_reset_instructions
-}
-trap print_rollback_instructions EXIT
-
-# All check are now complete, before we start alert if we are in dry-run
-if [[ $publish == 0 ]]; then
- echo "Performing dry-run, run with '-p' when you are ready to run and publish a release candidate"
-fi
-
-# This should be a clean repo we are working against. Run clean just to ensure it is.
-git clean -fdxq
-
-echo "Generating changelog"
-./dev/release/changelog $current_version
-git add CHANGELOG
-git commit -m "Updating CHANGELOG for ${current_version} release."
-
-echo "Committing updated .parquetcppversion on master"
-echo $new_snapshot_version > .parquetcppversion
-git add .parquetcppversion
-git commit -m "Incrementing snapshot version to ${new_snapshot_version}."
-
-echo "Creating ${rc_version} staging branch"
-git checkout -b "stage_${rc_version}"
-
-echo "Updating .parquetcppversion on staging branch"
-# Increment the version and create a branch
-echo ${current_version} > .parquetcppversion
-git add .parquetcppversion
-git commit -m "Updating .parquetcppversion to ${current_version}."
-
-# Build the source distribution from the new branch
-echo "Building the source distribution"
-dist_dir=dist
-dist_name="${version_tag}"
-
-mkdir -p ${dist_dir}
-git archive --prefix=${dist_name}/ -o ${dist_dir}/${dist_name}.tar.gz HEAD
-pushd ${dist_dir}
- # Sign the tarball.
- echo "Signing the distribution"
- gpg --armor --output ${dist_name}.tar.gz.asc --detach-sig ${dist_name}.tar.gz
-
- # Create the checksums
- echo "Creating checksums"
- gpg --print-md MD5 ${dist_name}.tar.gz > ${dist_name}.tar.gz.md5
- sha1sum ${dist_name}.tar.gz > ${dist_name}.tar.gz.sha1
- sha256sum ${dist_name}.tar.gz > ${dist_name}.tar.gz.sha256
- sha512sum ${dist_name}.tar.gz > ${dist_name}.tar.gz.sha512
-popd
-
-parquet_svn_rc_url="${parquet_svn_dist_url}/apache-parquet-cpp-${rc_version}"
-
-# Publish release candidate to svn and commit and push the new git tag
-if [[ $publish == 1 ]]; then
- echo "Publishing release candidate to ${parquet_svn_rc_url}"
- svn mkdir ${parquet_svn_rc_url} -m "parquet-cpp-${current_version} release candidate ${rc_version}"
- svn co --depth=empty ${parquet_svn_rc_url} ${dist_dir}
- pushd ${dist_dir}
- svn add ${dist_name}*
- svn ci -m "parquet-cpp-${current_version} release candidate ${rc_version}"
- popd
-
- echo "Creating tag ${rc_version_tag}"
- git tag -s ${rc_version_tag} \
- -m "Apache Parquet C++ ${current_version} release candidate ${rc_version}"
- git push origin "${rc_version_tag}"
-
- echo "Pushing updated .parquetcppversion to master"
- git checkout master
- git push origin "master:master-after-${rc_version_tag}"
-fi
-
-echo "Done creating the release candidate. The following draft email has been created"
-echo "to send to the dev@parquet.apache.org mailing list"
-echo
-
-# Create the email template for the release candidate to be sent to the mailing lists.
-if [[ $(uname) == Darwin ]]; then
- vote_end=$(date -v+3d)
-else
- vote_end=$(date -d+3days)
-fi
-
-MESSAGE=$(cat <<__EOF__
-To: dev@parquet.apache.org
-Subject: [VOTE] Release Apache Parquet C++ ${current_version} RC${rc_number}
-
-All,
-
-I propose that we accept the following release candidate as the official
-Apache Parquet C++ ${current_version} release.
-
-Parquet C++ ${rc_version} includes the following:
----
-The CHANGELOG for the release is available at:
-${parquetcpp_git_web_url}&f=CHANGELOG&hb=${rc_version_tag}
-
-The tag used to create the release candidate is:
-${parquetcpp_git_web_url};a=shortlog;h=refs/tags/${rc_version_tag}
-
-The release candidate is available at:
-${parquet_svn_rc_url}/${dist_name}.tar.gz
-
-The MD5 checksum of the release candidate can be found at:
-${parquet_svn_rc_url}/${dist_name}.tar.gz.md5
-
-The signature of the release candidate can be found at:
-${parquet_svn_rc_url}/${dist_name}.tar.gz.asc
-
-The GPG key used to sign the release are available at:
-${parquet_svn_dist_url}/KEYS
-
-The release is based on the commit hash ${current_git_rev}.
-
-Please download, verify, and test.
-
-The vote will close on ${vote_end}
-
-[ ] +1 Release this as Apache Parquet C++ ${current_version}
-[ ] +0
-[ ] -1 Do not release this as Apache Parquet C++ ${current_version} because...
-
-__EOF__
-)
-
-echo "--------------------------------------------------------------------------------"
-echo
-echo "${MESSAGE}"
-echo
-echo "--------------------------------------------------------------------------------"
-echo
-
-# Print reset instructions if this was a dry-run
-if [[ $publish == 0 ]]; then
- echo
- echo "This was a dry run, nothing has been published."
- echo
- print_reset_instructions
-fi
-
-# Unset error message handler and exit
-trap '' EXIT
-exit 0
diff --git a/dev/release/run-rat.sh b/dev/release/run-rat.sh
deleted file mode 100755
index 2f0d4ff..0000000
--- a/dev/release/run-rat.sh
+++ /dev/null
@@ -1,49 +0,0 @@
-#!/bin/bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-# download apache rat
-curl -s https://repo1.maven.org/maven2/org/apache/rat/apache-rat/0.12/apache-rat-0.12.jar > apache-rat-0.12.jar
-
-RAT="java -jar apache-rat-0.12.jar -d "
-
-# generate the rat report
-# Ignore symlinks as RAT does not seem to understand them.
-$RAT $1 \
- -e ".*" \
- -e mman.h \
- -e "*_generated.h" \
- -e asan_symbolize.py \
- -e cpplint.py \
- -e pax_global_header \
- -e clang_format_exclusions.txt \
- -e ArrowExternalProject.cmake \
- -e FindArrow.cmake \
- > rat.txt
-cat rat.txt
-UNAPPROVED=`cat rat.txt | grep "Unknown Licenses" | head -n 1 | cut -d " " -f 1`
-
-if [ "0" -eq "${UNAPPROVED}" ]; then
- echo "No unnaproved licenses"
-else
- echo "${UNAPPROVED} unapproved licences. Check rat report: rat.txt"
- exit 1
-fi
-
-
diff --git a/dev/release/verify-release-candidate b/dev/release/verify-release-candidate
deleted file mode 100755
index 8555b8a..0000000
--- a/dev/release/verify-release-candidate
+++ /dev/null
@@ -1,97 +0,0 @@
-#!/usr/bin/env bash
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-#
-# Downloads a release candidate and verifies that it passes binary
-# verification (signature and checksums) and test suites.
-#
-set -ex
-
-HERE=$(cd `dirname "${BASH_SOURCE[0]:-$0}"` && pwd)
-
-parquet_svn_dev_dist_url='https://dist.apache.org/repos/dist/dev/parquet'
-
-download_dist_file() {
- curl -f -O ${parquet_svn_dev_dist_url}/$1
-}
-
-download_rc_file() {
- download_dist_file apache-parquet-cpp-${verify_version}-rc${verify_rc_num}/$1
-}
-
-import_gpg_keys() {
- download_dist_file KEYS
- gpg --import KEYS
-}
-
-fetch_archive() {
- local dist_name=$1
- download_rc_file ${dist_name}.tar.gz
- download_rc_file ${dist_name}.tar.gz.asc
- download_rc_file ${dist_name}.tar.gz.md5
- download_rc_file ${dist_name}.tar.gz.sha512
- gpg --verify ${dist_name}.tar.gz.asc ${dist_name}.tar.gz
- gpg --print-md MD5 ${dist_name}.tar.gz | diff - ${dist_name}.tar.gz.md5
- if [ "$(uname)" == "Darwin" ]; then
- shasum -a 512 ${dist_name}.tar.gz | diff - ${dist_name}.tar.gz.sha512
- else
- sha512sum ${dist_name}.tar.gz | diff - ${dist_name}.tar.gz.sha512
- fi
-}
-
-run_tests() {
- # Build
- mkdir -p build
- cd build
- cmake ..
- make -j5
-
- # Test
- export PARQUET_TEST_DATA=$(dirname $(pwd))/data
- make unittest
-}
-
-setup_tempdir() {
- cleanup() {
- rm -fr "$TMPDIR"
- }
- trap cleanup EXIT
- TMPDIR=$(mktemp -d -t "$1.XXXXX")
-}
-
-case $# in
- 2) verify_version="$1"
- verify_rc_num="$2"
- ;;
-
- *) echo "Usage: $0 VERSION RC_NUM"
- echo "Example to test 1.3.0-rc2: $0 1.3.0 2"
- exit 1
- ;;
-esac
-
-setup_tempdir "parquet-cpp-$verify_version"
-echo "Working in sandbox $TMPDIR"
-cd $TMPDIR
-
-import_gpg_keys
-
-dist_name="apache-parquet-cpp-${verify_version}"
-fetch_archive $dist_name
-tar xvzf ${dist_name}.tar.gz
-cd ${dist_name}
-run_tests
-
-echo 'Release candidate looks good!'
-exit 0
diff --git a/dev/release/verify-release-candidate.bat b/dev/release/verify-release-candidate.bat
deleted file mode 100644
index b5aaa8a..0000000
--- a/dev/release/verify-release-candidate.bat
+++ /dev/null
@@ -1,54 +0,0 @@
-@rem Licensed to the Apache Software Foundation (ASF) under one
-@rem or more contributor license agreements. See the NOTICE file
-@rem distributed with this work for additional information
-@rem regarding copyright ownership. The ASF licenses this file
-@rem to you under the Apache License, Version 2.0 (the
-@rem "License"); you may not use this file except in compliance
-@rem with the License. You may obtain a copy of the License at
-@rem
-@rem http://www.apache.org/licenses/LICENSE-2.0
-@rem
-@rem Unless required by applicable law or agreed to in writing,
-@rem software distributed under the License is distributed on an
-@rem "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-@rem KIND, either express or implied. See the License for the
-@rem specific language governing permissions and limitations
-@rem under the License.
-
-@rem Then run from the directory containing the RC tarball
-@rem
-@rem verify-release-candidate.bat apache-parquet-cpp-%VERSION%
-
-@echo on
-
-if not exist "C:\tmp\" mkdir C:\tmp
-if exist "C:\tmp\parquet-verify-release" rd C:\tmp\parquet-verify-release /s /q
-if not exist "C:\tmp\parquet-verify-release" mkdir C:\tmp\parquet-verify-release
-
-tar xvf %1.tar.gz -C "C:/tmp/"
-
-set GENERATOR=Visual Studio 14 2015 Win64
-set CONFIGURATION=release
-set PARQUET_SOURCE=C:\tmp\%1
-set INSTALL_DIR=C:\tmp\%1\install
-
-pushd %PARQUET_SOURCE%
-
-set PARQUET_TEST_DATA=%PARQUET_SOURCE%\data
-
-set PARQUET_HOME=%INSTALL_DIR%
-set PATH=%INSTALL_DIR%\bin;%PATH%
-
-mkdir build
-pushd build
-
-cmake -G "%GENERATOR%" ^
- -DCMAKE_INSTALL_PREFIX=%PARQUET_HOME% ^
- -DPARQUET_BOOST_USE_SHARED=OFF ^
- -DCMAKE_BUILD_TYPE=%CONFIGURATION% ^
- -DPARQUET_CXXFLAGS="/WX /MP" ^
- .. || exit /B
-cmake --build . --target INSTALL --config %CONFIGURATION% || exit /B
-
-ctest -VV || exit /B
-popd
diff --git a/docs/Windows.md b/docs/Windows.md
deleted file mode 100644
index 8de04f6..0000000
--- a/docs/Windows.md
+++ /dev/null
@@ -1,142 +0,0 @@
-<!---
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License. See accompanying LICENSE file.
--->
-
-# Building parquet-cpp on Windows
-
-## Fast setup of building requirements with conda and conda-forge
-
-A convenient and tested way to set up thirdparty dependencies is to use the
-conda package manager.
-Please feel free to extend this document with others ways to setup
-development environment for parquet-cpp.
-
-### conda and package toolchain
-
-[Miniconda][1] is a minimal Python distribution including the conda package
-manager. To get started, download and install a 64-bit distribution.
-
-We recommend using packages from [conda-forge][2].
-Launch cmd.exe and run following to bootstrap a build environment:
-
-```shell
-conda create -n parquet-dev cmake git boost-cpp curl zlib snappy brotli thrift-cpp -c conda-forge
-```
-
-To allow cmake to pick up 3rd party dependencies, you should set
-`PARQUET_BUILD_TOOLCHAIN` environment variable to contain `Library` folder
-path of new created on previous step `parquet-dev` conda environment.
-For instance, if `Miniconda` was installed to default destination, `Library`
-folder path for `parquet-dev` conda environment will be as following:
-
-```shell
-C:\Users\YOUR_USER_NAME\Miniconda3\envs\parquet-dev\Library
-```
-
-As alternative to `PARQUET_BUILD_TOOLCHAIN`, it's possible to configure path
-to each 3rd party dependency separately by setting appropriate environment
-variable:
-
-`BOOST_ROOT` variable with path to `boost` installation
-`THRIFT_HOME` variable with path to `thrift-cpp` installation
-`SNAPPY_HOME` variable with path to `snappy` installation
-`ZLIB_HOME` variable with path to `zlib` installation
-`BROTLI_HOME` variable with path to `brotli` installation
-`ARROW_HOME` variable with path to `arrow` installation
-
-### Customize static libraries names lookup of 3rd party dependencies
-
-If you decided to use pre-built 3rd party dependencies libs, it's possible to
-configure parquet-cpp cmake build script to search for customized names of 3rd
-party static libs.
-
-`zlib`. Pass `-DPARQUET_ZLIB_VENDORED=OFF` to enable lookup of custom zlib
-build. Set `ZLIB_HOME` environment variable. Pass
-`-DZLIB_MSVC_STATIC_LIB_SUFFIX=%ZLIB_SUFFIX%` to link with z%ZLIB_SUFFIX%.lib
-
-`arrow`. Set `ARROW_HOME` environment variable. Pass
-`-DARROW_MSVC_STATIC_LIB_SUFFIX=%ARROW_SUFFIX%` to link with
-arrow%ARROW_SUFFIX%.lib
-
-`brotli`. Set `BROTLY_HOME` environment variable. Pass
-`-DBROTLI_MSVC_STATIC_LIB_SUFFIX=%BROTLI_SUFFIX%` to link with
-brotli*%BROTLI_SUFFIX%.lib.
-
-`snappy`. Set `SNAPPY_HOME` environment variable. Pass
-`-DSNAPPY_MSVC_STATIC_LIB_SUFFIX=%SNAPPY_SUFFIX%` to link with
-snappy%SNAPPY_SUFFIX%.lib.
-
-`thrift`. Set `THRIFT_HOME` environment variable. Pass
-`-DTHRIFT_MSVC_STATIC_LIB_SUFFIX=%THRIFT_SUFFIX%` to link with
-thrift*%THRIFT_SUFFIX%.lib.
-
-### Visual Studio
-
-Microsoft provides the free Visual Studio Community edition. Once you have
-Visual Studio installed, you should configure cmd.exe environment to be able
-to find Visual Studio's build toolchain by running following commands:
-
-#### Visual Studio 2015
-
-```"C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat" amd64```
-
-#### Visual Studio 2017
-
-```"C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\Common7\Tools\VsDevCmd.bat" -arch=amd64```
-
-It's easiest to configure a console emulator like [cmder][3] to automatically
-launch this when starting a new development console.
-
-## Building with NMake
-
-Activate your conda build environment:
-
-```
-activate parquet-dev
-```
-
-Change working directory in cmd.exe to the root directory of parquet-cpp and
-do an out of source build using `nmake`:
-
-```
-cd %PARQUET_ROOT_SOURCES_DIRECTORY%
-mkdir build
-cd build
-cmake -G "NMake Makefiles" -DCMAKE_BUILD_TYPE=Release ..
-nmake
-```
-
-## Building with Visual Studio cmake generator
-
-Activate your conda build environment:
-
-```
-activate parquet-dev
-```
-
-Change working directory in cmd.exe to the root directory of parquet-cpp and
-do an out of source build:
-
-```
-cd %PARQUET_ROOT_SOURCES_DIRECTORY%
-mkdir build
-cd build
-cmake -G "Visual Studio 14 2015 Win64" -DCMAKE_BUILD_TYPE=Release ..
-cmake --build . --config Release
-```
-
-When using conda, only release builds are currently supported.
-
-[1]: https://conda.io/miniconda.html
-[2]: https://conda-forge.github.io/
-[3]: http://cmder.net/
\ No newline at end of file
diff --git a/examples/low-level-api/CMakeLists.txt b/examples/low-level-api/CMakeLists.txt
deleted file mode 100644
index 64ba110..0000000
--- a/examples/low-level-api/CMakeLists.txt
+++ /dev/null
@@ -1,25 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-if (PARQUET_BUILD_EXECUTABLES)
- add_executable(reader-writer reader-writer.cc)
- add_executable(reader-writer2 reader-writer2.cc)
- target_include_directories(reader-writer PRIVATE .)
- target_include_directories(reader-writer2 PRIVATE .)
- target_link_libraries(reader-writer parquet_static)
- target_link_libraries(reader-writer2 parquet_static)
-endif()
diff --git a/examples/low-level-api/reader-writer.cc b/examples/low-level-api/reader-writer.cc
deleted file mode 100644
index 09cd137..0000000
--- a/examples/low-level-api/reader-writer.cc
+++ /dev/null
@@ -1,409 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#include <cassert>
-#include <fstream>
-#include <iostream>
-#include <memory>
-
-#include <reader_writer.h>
-
-/*
- * This example describes writing and reading Parquet Files in C++ and serves as a
- * reference to the API.
- * The file contains all the physical data types supported by Parquet.
- * This example uses the RowGroupWriter API that supports writing RowGroups optimized for
- *memory consumption
- **/
-
-/* Parquet is a structured columnar file format
- * Parquet File = "Parquet data" + "Parquet Metadata"
- * "Parquet data" is simply a vector of RowGroups. Each RowGroup is a batch of rows in a
- * columnar layout
- * "Parquet Metadata" contains the "file schema" and attributes of the RowGroups and their
- * Columns
- * "file schema" is a tree where each node is either a primitive type (leaf nodes) or a
- * complex (nested) type (internal nodes)
- * For specific details, please refer the format here:
- * https://github.com/apache/parquet-format/blob/master/LogicalTypes.md
- **/
-
-constexpr int NUM_ROWS_PER_ROW_GROUP = 500;
-const char PARQUET_FILENAME[] = "parquet_cpp_example.parquet";
-
-int main(int argc, char** argv) {
- /**********************************************************************************
- PARQUET WRITER EXAMPLE
- **********************************************************************************/
- // parquet::REQUIRED fields do not need definition and repetition level values
- // parquet::OPTIONAL fields require only definition level values
- // parquet::REPEATED fields require both definition and repetition level values
- try {
- // Create a local file output stream instance.
- using FileClass = ::arrow::io::FileOutputStream;
- std::shared_ptr<FileClass> out_file;
- PARQUET_THROW_NOT_OK(FileClass::Open(PARQUET_FILENAME, &out_file));
-
- // Setup the parquet schema
- std::shared_ptr<GroupNode> schema = SetupSchema();
-
- // Add writer properties
- parquet::WriterProperties::Builder builder;
- builder.compression(parquet::Compression::SNAPPY);
- std::shared_ptr<parquet::WriterProperties> props = builder.build();
-
- // Create a ParquetFileWriter instance
- std::shared_ptr<parquet::ParquetFileWriter> file_writer =
- parquet::ParquetFileWriter::Open(out_file, schema, props);
-
- // Append a RowGroup with a specific number of rows.
- parquet::RowGroupWriter* rg_writer = file_writer->AppendRowGroup();
-
- // Write the Bool column
- parquet::BoolWriter* bool_writer =
- static_cast<parquet::BoolWriter*>(rg_writer->NextColumn());
- for (int i = 0; i < NUM_ROWS_PER_ROW_GROUP; i++) {
- bool value = ((i % 2) == 0) ? true : false;
- bool_writer->WriteBatch(1, nullptr, nullptr, &value);
- }
-
- // Write the Int32 column
- parquet::Int32Writer* int32_writer =
- static_cast<parquet::Int32Writer*>(rg_writer->NextColumn());
- for (int i = 0; i < NUM_ROWS_PER_ROW_GROUP; i++) {
- int32_t value = i;
- int32_writer->WriteBatch(1, nullptr, nullptr, &value);
- }
-
- // Write the Int64 column. Each row has repeats twice.
- parquet::Int64Writer* int64_writer =
- static_cast<parquet::Int64Writer*>(rg_writer->NextColumn());
- for (int i = 0; i < 2 * NUM_ROWS_PER_ROW_GROUP; i++) {
- int64_t value = i * 1000 * 1000;
- value *= 1000 * 1000;
- int16_t definition_level = 1;
- int16_t repetition_level = 0;
- if ((i % 2) == 0) {
- repetition_level = 1; // start of a new record
- }
- int64_writer->WriteBatch(1, &definition_level, &repetition_level, &value);
- }
-
- // Write the INT96 column.
- parquet::Int96Writer* int96_writer =
- static_cast<parquet::Int96Writer*>(rg_writer->NextColumn());
- for (int i = 0; i < NUM_ROWS_PER_ROW_GROUP; i++) {
- parquet::Int96 value;
- value.value[0] = i;
- value.value[1] = i + 1;
- value.value[2] = i + 2;
- int96_writer->WriteBatch(1, nullptr, nullptr, &value);
- }
-
- // Write the Float column
- parquet::FloatWriter* float_writer =
- static_cast<parquet::FloatWriter*>(rg_writer->NextColumn());
- for (int i = 0; i < NUM_ROWS_PER_ROW_GROUP; i++) {
- float value = static_cast<float>(i) * 1.1f;
- float_writer->WriteBatch(1, nullptr, nullptr, &value);
- }
-
- // Write the Double column
- parquet::DoubleWriter* double_writer =
- static_cast<parquet::DoubleWriter*>(rg_writer->NextColumn());
- for (int i = 0; i < NUM_ROWS_PER_ROW_GROUP; i++) {
- double value = i * 1.1111111;
- double_writer->WriteBatch(1, nullptr, nullptr, &value);
- }
-
- // Write the ByteArray column. Make every alternate values NULL
- parquet::ByteArrayWriter* ba_writer =
- static_cast<parquet::ByteArrayWriter*>(rg_writer->NextColumn());
- for (int i = 0; i < NUM_ROWS_PER_ROW_GROUP; i++) {
- parquet::ByteArray value;
- char hello[FIXED_LENGTH] = "parquet";
- hello[7] = static_cast<char>(static_cast<int>('0') + i / 100);
- hello[8] = static_cast<char>(static_cast<int>('0') + (i / 10) % 10);
- hello[9] = static_cast<char>(static_cast<int>('0') + i % 10);
- if (i % 2 == 0) {
- int16_t definition_level = 1;
- value.ptr = reinterpret_cast<const uint8_t*>(&hello[0]);
- value.len = FIXED_LENGTH;
- ba_writer->WriteBatch(1, &definition_level, nullptr, &value);
- } else {
- int16_t definition_level = 0;
- ba_writer->WriteBatch(1, &definition_level, nullptr, nullptr);
- }
- }
-
- // Write the FixedLengthByteArray column
- parquet::FixedLenByteArrayWriter* flba_writer =
- static_cast<parquet::FixedLenByteArrayWriter*>(rg_writer->NextColumn());
- for (int i = 0; i < NUM_ROWS_PER_ROW_GROUP; i++) {
- parquet::FixedLenByteArray value;
- char v = static_cast<char>(i);
- char flba[FIXED_LENGTH] = {v, v, v, v, v, v, v, v, v, v};
- value.ptr = reinterpret_cast<const uint8_t*>(&flba[0]);
-
- flba_writer->WriteBatch(1, nullptr, nullptr, &value);
- }
-
- // Close the ParquetFileWriter
- file_writer->Close();
-
- // Write the bytes to file
- DCHECK(out_file->Close().ok());
- } catch (const std::exception& e) {
- std::cerr << "Parquet write error: " << e.what() << std::endl;
- return -1;
- }
-
- /**********************************************************************************
- PARQUET READER EXAMPLE
- **********************************************************************************/
-
- try {
- // Create a ParquetReader instance
- std::unique_ptr<parquet::ParquetFileReader> parquet_reader =
- parquet::ParquetFileReader::OpenFile(PARQUET_FILENAME, false);
-
- // Get the File MetaData
- std::shared_ptr<parquet::FileMetaData> file_metadata = parquet_reader->metadata();
-
- // Get the number of RowGroups
- int num_row_groups = file_metadata->num_row_groups();
- assert(num_row_groups == 1);
-
- // Get the number of Columns
- int num_columns = file_metadata->num_columns();
- assert(num_columns == 8);
-
- // Iterate over all the RowGroups in the file
- for (int r = 0; r < num_row_groups; ++r) {
- // Get the RowGroup Reader
- std::shared_ptr<parquet::RowGroupReader> row_group_reader =
- parquet_reader->RowGroup(r);
-
- int64_t values_read = 0;
- int64_t rows_read = 0;
- int16_t definition_level;
- int16_t repetition_level;
- int i;
- std::shared_ptr<parquet::ColumnReader> column_reader;
-
- // Get the Column Reader for the boolean column
- column_reader = row_group_reader->Column(0);
- parquet::BoolReader* bool_reader =
- static_cast<parquet::BoolReader*>(column_reader.get());
-
- // Read all the rows in the column
- i = 0;
- while (bool_reader->HasNext()) {
- bool value;
- // Read one value at a time. The number of rows read is returned. values_read
- // contains the number of non-null rows
- rows_read = bool_reader->ReadBatch(1, nullptr, nullptr, &value, &values_read);
- // Ensure only one value is read
- assert(rows_read == 1);
- // There are no NULL values in the rows written
- assert(values_read == 1);
- // Verify the value written
- bool expected_value = ((i % 2) == 0) ? true : false;
- assert(value == expected_value);
- i++;
- }
-
- // Get the Column Reader for the Int32 column
- column_reader = row_group_reader->Column(1);
- parquet::Int32Reader* int32_reader =
- static_cast<parquet::Int32Reader*>(column_reader.get());
- // Read all the rows in the column
- i = 0;
- while (int32_reader->HasNext()) {
- int32_t value;
- // Read one value at a time. The number of rows read is returned. values_read
- // contains the number of non-null rows
- rows_read = int32_reader->ReadBatch(1, nullptr, nullptr, &value, &values_read);
- // Ensure only one value is read
- assert(rows_read == 1);
- // There are no NULL values in the rows written
- assert(values_read == 1);
- // Verify the value written
- assert(value == i);
- i++;
- }
-
- // Get the Column Reader for the Int64 column
- column_reader = row_group_reader->Column(2);
- parquet::Int64Reader* int64_reader =
- static_cast<parquet::Int64Reader*>(column_reader.get());
- // Read all the rows in the column
- i = 0;
- while (int64_reader->HasNext()) {
- int64_t value;
- // Read one value at a time. The number of rows read is returned. values_read
- // contains the number of non-null rows
- rows_read = int64_reader->ReadBatch(1, &definition_level, &repetition_level,
- &value, &values_read);
- // Ensure only one value is read
- assert(rows_read == 1);
- // There are no NULL values in the rows written
- assert(values_read == 1);
- // Verify the value written
- int64_t expected_value = i * 1000 * 1000;
- expected_value *= 1000 * 1000;
- assert(value == expected_value);
- if ((i % 2) == 0) {
- assert(repetition_level == 1);
- } else {
- assert(repetition_level == 0);
- }
- i++;
- }
-
- // Get the Column Reader for the Int96 column
- column_reader = row_group_reader->Column(3);
- parquet::Int96Reader* int96_reader =
- static_cast<parquet::Int96Reader*>(column_reader.get());
- // Read all the rows in the column
- i = 0;
- while (int96_reader->HasNext()) {
- parquet::Int96 value;
- // Read one value at a time. The number of rows read is returned. values_read
- // contains the number of non-null rows
- rows_read = int96_reader->ReadBatch(1, nullptr, nullptr, &value, &values_read);
- // Ensure only one value is read
- assert(rows_read == 1);
- // There are no NULL values in the rows written
- assert(values_read == 1);
- // Verify the value written
- parquet::Int96 expected_value;
- expected_value.value[0] = i;
- expected_value.value[1] = i + 1;
- expected_value.value[2] = i + 2;
- for (int j = 0; j < 3; j++) {
- assert(value.value[j] == expected_value.value[j]);
- }
- i++;
- }
-
- // Get the Column Reader for the Float column
- column_reader = row_group_reader->Column(4);
- parquet::FloatReader* float_reader =
- static_cast<parquet::FloatReader*>(column_reader.get());
- // Read all the rows in the column
- i = 0;
- while (float_reader->HasNext()) {
- float value;
- // Read one value at a time. The number of rows read is returned. values_read
- // contains the number of non-null rows
- rows_read = float_reader->ReadBatch(1, nullptr, nullptr, &value, &values_read);
- // Ensure only one value is read
- assert(rows_read == 1);
- // There are no NULL values in the rows written
- assert(values_read == 1);
- // Verify the value written
- float expected_value = static_cast<float>(i) * 1.1f;
- assert(value == expected_value);
- i++;
- }
-
- // Get the Column Reader for the Double column
- column_reader = row_group_reader->Column(5);
- parquet::DoubleReader* double_reader =
- static_cast<parquet::DoubleReader*>(column_reader.get());
- // Read all the rows in the column
- i = 0;
- while (double_reader->HasNext()) {
- double value;
- // Read one value at a time. The number of rows read is returned. values_read
- // contains the number of non-null rows
- rows_read = double_reader->ReadBatch(1, nullptr, nullptr, &value, &values_read);
- // Ensure only one value is read
- assert(rows_read == 1);
- // There are no NULL values in the rows written
- assert(values_read == 1);
- // Verify the value written
- double expected_value = i * 1.1111111;
- assert(value == expected_value);
- i++;
- }
-
- // Get the Column Reader for the ByteArray column
- column_reader = row_group_reader->Column(6);
- parquet::ByteArrayReader* ba_reader =
- static_cast<parquet::ByteArrayReader*>(column_reader.get());
- // Read all the rows in the column
- i = 0;
- while (ba_reader->HasNext()) {
- parquet::ByteArray value;
- // Read one value at a time. The number of rows read is returned. values_read
- // contains the number of non-null rows
- rows_read =
- ba_reader->ReadBatch(1, &definition_level, nullptr, &value, &values_read);
- // Ensure only one value is read
- assert(rows_read == 1);
- // Verify the value written
- char expected_value[FIXED_LENGTH] = "parquet";
- expected_value[7] = static_cast<char>('0' + i / 100);
- expected_value[8] = static_cast<char>('0' + (i / 10) % 10);
- expected_value[9] = static_cast<char>('0' + i % 10);
- if (i % 2 == 0) { // only alternate values exist
- // There are no NULL values in the rows written
- assert(values_read == 1);
- assert(value.len == FIXED_LENGTH);
- assert(memcmp(value.ptr, &expected_value[0], FIXED_LENGTH) == 0);
- assert(definition_level == 1);
- } else {
- // There are NULL values in the rows written
- assert(values_read == 0);
- assert(definition_level == 0);
- }
- i++;
- }
-
- // Get the Column Reader for the FixedLengthByteArray column
- column_reader = row_group_reader->Column(7);
- parquet::FixedLenByteArrayReader* flba_reader =
- static_cast<parquet::FixedLenByteArrayReader*>(column_reader.get());
- // Read all the rows in the column
- i = 0;
- while (flba_reader->HasNext()) {
- parquet::FixedLenByteArray value;
- // Read one value at a time. The number of rows read is returned. values_read
- // contains the number of non-null rows
- rows_read = flba_reader->ReadBatch(1, nullptr, nullptr, &value, &values_read);
- // Ensure only one value is read
- assert(rows_read == 1);
- // There are no NULL values in the rows written
- assert(values_read == 1);
- // Verify the value written
- char v = static_cast<char>(i);
- char expected_value[FIXED_LENGTH] = {v, v, v, v, v, v, v, v, v, v};
- assert(memcmp(value.ptr, &expected_value[0], FIXED_LENGTH) == 0);
- i++;
- }
- }
- } catch (const std::exception& e) {
- std::cerr << "Parquet read error: " << e.what() << std::endl;
- return -1;
- }
-
- std::cout << "Parquet Writing and Reading Complete" << std::endl;
-
- return 0;
-}
diff --git a/examples/low-level-api/reader-writer2.cc b/examples/low-level-api/reader-writer2.cc
deleted file mode 100644
index dded5fa..0000000
--- a/examples/low-level-api/reader-writer2.cc
+++ /dev/null
@@ -1,430 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#include <cassert>
-#include <fstream>
-#include <iostream>
-#include <memory>
-
-#include <reader_writer.h>
-
-/*
- * This example describes writing and reading Parquet Files in C++ and serves as a
- * reference to the API.
- * The file contains all the physical data types supported by Parquet.
- * This example uses the RowGroupWriter API that supports writing RowGroups based on a
- *certain size
- **/
-
-/* Parquet is a structured columnar file format
- * Parquet File = "Parquet data" + "Parquet Metadata"
- * "Parquet data" is simply a vector of RowGroups. Each RowGroup is a batch of rows in a
- * columnar layout
- * "Parquet Metadata" contains the "file schema" and attributes of the RowGroups and their
- * Columns
- * "file schema" is a tree where each node is either a primitive type (leaf nodes) or a
- * complex (nested) type (internal nodes)
- * For specific details, please refer the format here:
- * https://github.com/apache/parquet-format/blob/master/LogicalTypes.md
- **/
-
-constexpr int NUM_ROWS = 2500000;
-constexpr int64_t ROW_GROUP_SIZE = 16 * 1024 * 1024; // 16 MB
-const char PARQUET_FILENAME[] = "parquet_cpp_example2.parquet";
-
-int main(int argc, char** argv) {
- /**********************************************************************************
- PARQUET WRITER EXAMPLE
- **********************************************************************************/
- // parquet::REQUIRED fields do not need definition and repetition level values
- // parquet::OPTIONAL fields require only definition level values
- // parquet::REPEATED fields require both definition and repetition level values
- try {
- // Create a local file output stream instance.
- using FileClass = ::arrow::io::FileOutputStream;
- std::shared_ptr<FileClass> out_file;
- PARQUET_THROW_NOT_OK(FileClass::Open(PARQUET_FILENAME, &out_file));
-
- // Setup the parquet schema
- std::shared_ptr<GroupNode> schema = SetupSchema();
-
- // Add writer properties
- parquet::WriterProperties::Builder builder;
- builder.compression(parquet::Compression::SNAPPY);
- std::shared_ptr<parquet::WriterProperties> props = builder.build();
-
- // Create a ParquetFileWriter instance
- std::shared_ptr<parquet::ParquetFileWriter> file_writer =
- parquet::ParquetFileWriter::Open(out_file, schema, props);
-
- // Append a BufferedRowGroup to keep the RowGroup open until a certain size
- parquet::RowGroupWriter* rg_writer = file_writer->AppendBufferedRowGroup();
-
- int num_columns = file_writer->num_columns();
- std::vector<int64_t> buffered_values_estimate(num_columns, 0);
- for (int i = 0; i < NUM_ROWS; i++) {
- int64_t estimated_bytes = 0;
- // Get the estimated size of the values that are not written to a page yet
- for (int n = 0; n < num_columns; n++) {
- estimated_bytes += buffered_values_estimate[n];
- }
-
- // We need to consider the compressed pages
- // as well as the values that are not compressed yet
- if ((rg_writer->total_bytes_written() + rg_writer->total_compressed_bytes() +
- estimated_bytes) > ROW_GROUP_SIZE) {
- rg_writer->Close();
- std::fill(buffered_values_estimate.begin(), buffered_values_estimate.end(), 0);
- rg_writer = file_writer->AppendBufferedRowGroup();
- }
-
- int col_id = 0;
- // Write the Bool column
- parquet::BoolWriter* bool_writer =
- static_cast<parquet::BoolWriter*>(rg_writer->column(col_id));
- bool bool_value = ((i % 2) == 0) ? true : false;
- bool_writer->WriteBatch(1, nullptr, nullptr, &bool_value);
- buffered_values_estimate[col_id] = bool_writer->EstimatedBufferedValueBytes();
-
- // Write the Int32 column
- col_id++;
- parquet::Int32Writer* int32_writer =
- static_cast<parquet::Int32Writer*>(rg_writer->column(col_id));
- int32_t int32_value = i;
- int32_writer->WriteBatch(1, nullptr, nullptr, &int32_value);
- buffered_values_estimate[col_id] = int32_writer->EstimatedBufferedValueBytes();
-
- // Write the Int64 column. Each row has repeats twice.
- col_id++;
- parquet::Int64Writer* int64_writer =
- static_cast<parquet::Int64Writer*>(rg_writer->column(col_id));
- int64_t int64_value1 = 2 * i;
- int16_t definition_level = 1;
- int16_t repetition_level = 0;
- int64_writer->WriteBatch(1, &definition_level, &repetition_level, &int64_value1);
- int64_t int64_value2 = (2 * i + 1);
- repetition_level = 1; // start of a new record
- int64_writer->WriteBatch(1, &definition_level, &repetition_level, &int64_value2);
- buffered_values_estimate[col_id] = int64_writer->EstimatedBufferedValueBytes();
-
- // Write the INT96 column.
- col_id++;
- parquet::Int96Writer* int96_writer =
- static_cast<parquet::Int96Writer*>(rg_writer->column(col_id));
- parquet::Int96 int96_value;
- int96_value.value[0] = i;
- int96_value.value[1] = i + 1;
- int96_value.value[2] = i + 2;
- int96_writer->WriteBatch(1, nullptr, nullptr, &int96_value);
- buffered_values_estimate[col_id] = int96_writer->EstimatedBufferedValueBytes();
-
- // Write the Float column
- col_id++;
- parquet::FloatWriter* float_writer =
- static_cast<parquet::FloatWriter*>(rg_writer->column(col_id));
- float float_value = static_cast<float>(i) * 1.1f;
- float_writer->WriteBatch(1, nullptr, nullptr, &float_value);
- buffered_values_estimate[col_id] = float_writer->EstimatedBufferedValueBytes();
-
- // Write the Double column
- col_id++;
- parquet::DoubleWriter* double_writer =
- static_cast<parquet::DoubleWriter*>(rg_writer->column(col_id));
- double double_value = i * 1.1111111;
- double_writer->WriteBatch(1, nullptr, nullptr, &double_value);
- buffered_values_estimate[col_id] = double_writer->EstimatedBufferedValueBytes();
-
- // Write the ByteArray column. Make every alternate values NULL
- col_id++;
- parquet::ByteArrayWriter* ba_writer =
- static_cast<parquet::ByteArrayWriter*>(rg_writer->column(col_id));
- parquet::ByteArray ba_value;
- char hello[FIXED_LENGTH] = "parquet";
- hello[7] = static_cast<char>(static_cast<int>('0') + i / 100);
- hello[8] = static_cast<char>(static_cast<int>('0') + (i / 10) % 10);
- hello[9] = static_cast<char>(static_cast<int>('0') + i % 10);
- if (i % 2 == 0) {
- int16_t definition_level = 1;
- ba_value.ptr = reinterpret_cast<const uint8_t*>(&hello[0]);
- ba_value.len = FIXED_LENGTH;
- ba_writer->WriteBatch(1, &definition_level, nullptr, &ba_value);
- } else {
- int16_t definition_level = 0;
- ba_writer->WriteBatch(1, &definition_level, nullptr, nullptr);
- }
- buffered_values_estimate[col_id] = ba_writer->EstimatedBufferedValueBytes();
-
- // Write the FixedLengthByteArray column
- col_id++;
- parquet::FixedLenByteArrayWriter* flba_writer =
- static_cast<parquet::FixedLenByteArrayWriter*>(rg_writer->column(col_id));
- parquet::FixedLenByteArray flba_value;
- char v = static_cast<char>(i);
- char flba[FIXED_LENGTH] = {v, v, v, v, v, v, v, v, v, v};
- flba_value.ptr = reinterpret_cast<const uint8_t*>(&flba[0]);
-
- flba_writer->WriteBatch(1, nullptr, nullptr, &flba_value);
- buffered_values_estimate[col_id] = flba_writer->EstimatedBufferedValueBytes();
- }
-
- // Close the RowGroupWriter
- rg_writer->Close();
- // Close the ParquetFileWriter
- file_writer->Close();
-
- // Write the bytes to file
- DCHECK(out_file->Close().ok());
- } catch (const std::exception& e) {
- std::cerr << "Parquet write error: " << e.what() << std::endl;
- return -1;
- }
-
- /**********************************************************************************
- PARQUET READER EXAMPLE
- **********************************************************************************/
-
- try {
- // Create a ParquetReader instance
- std::unique_ptr<parquet::ParquetFileReader> parquet_reader =
- parquet::ParquetFileReader::OpenFile(PARQUET_FILENAME, false);
-
- // Get the File MetaData
- std::shared_ptr<parquet::FileMetaData> file_metadata = parquet_reader->metadata();
-
- int num_row_groups = file_metadata->num_row_groups();
-
- // Get the number of Columns
- int num_columns = file_metadata->num_columns();
- assert(num_columns == 8);
-
- std::vector<int> col_row_counts(num_columns, 0);
-
- // Iterate over all the RowGroups in the file
- for (int r = 0; r < num_row_groups; ++r) {
- // Get the RowGroup Reader
- std::shared_ptr<parquet::RowGroupReader> row_group_reader =
- parquet_reader->RowGroup(r);
-
- assert(row_group_reader->metadata()->total_byte_size() < ROW_GROUP_SIZE);
-
- int64_t values_read = 0;
- int64_t rows_read = 0;
- int16_t definition_level;
- int16_t repetition_level;
- std::shared_ptr<parquet::ColumnReader> column_reader;
- int col_id = 0;
-
- // Get the Column Reader for the boolean column
- column_reader = row_group_reader->Column(col_id);
- parquet::BoolReader* bool_reader =
- static_cast<parquet::BoolReader*>(column_reader.get());
-
- // Read all the rows in the column
- while (bool_reader->HasNext()) {
- bool value;
- // Read one value at a time. The number of rows read is returned. values_read
- // contains the number of non-null rows
- rows_read = bool_reader->ReadBatch(1, nullptr, nullptr, &value, &values_read);
- // Ensure only one value is read
- assert(rows_read == 1);
- // There are no NULL values in the rows written
- assert(values_read == 1);
- // Verify the value written
- bool expected_value = ((col_row_counts[col_id] % 2) == 0) ? true : false;
- assert(value == expected_value);
- col_row_counts[col_id]++;
- }
-
- // Get the Column Reader for the Int32 column
- col_id++;
- column_reader = row_group_reader->Column(col_id);
- parquet::Int32Reader* int32_reader =
- static_cast<parquet::Int32Reader*>(column_reader.get());
- // Read all the rows in the column
- while (int32_reader->HasNext()) {
- int32_t value;
- // Read one value at a time. The number of rows read is returned. values_read
- // contains the number of non-null rows
- rows_read = int32_reader->ReadBatch(1, nullptr, nullptr, &value, &values_read);
- // Ensure only one value is read
- assert(rows_read == 1);
- // There are no NULL values in the rows written
- assert(values_read == 1);
- // Verify the value written
- assert(value == col_row_counts[col_id]);
- col_row_counts[col_id]++;
- }
-
- // Get the Column Reader for the Int64 column
- col_id++;
- column_reader = row_group_reader->Column(col_id);
- parquet::Int64Reader* int64_reader =
- static_cast<parquet::Int64Reader*>(column_reader.get());
- // Read all the rows in the column
- while (int64_reader->HasNext()) {
- int64_t value;
- // Read one value at a time. The number of rows read is returned. values_read
- // contains the number of non-null rows
- rows_read = int64_reader->ReadBatch(1, &definition_level, &repetition_level,
- &value, &values_read);
- // Ensure only one value is read
- assert(rows_read == 1);
- // There are no NULL values in the rows written
- assert(values_read == 1);
- // Verify the value written
- int64_t expected_value = col_row_counts[col_id];
- assert(value == expected_value);
- if ((col_row_counts[col_id] % 2) == 0) {
- assert(repetition_level == 0);
- } else {
- assert(repetition_level == 1);
- }
- col_row_counts[col_id]++;
- }
-
- // Get the Column Reader for the Int96 column
- col_id++;
- column_reader = row_group_reader->Column(col_id);
- parquet::Int96Reader* int96_reader =
- static_cast<parquet::Int96Reader*>(column_reader.get());
- // Read all the rows in the column
- while (int96_reader->HasNext()) {
- parquet::Int96 value;
- // Read one value at a time. The number of rows read is returned. values_read
- // contains the number of non-null rows
- rows_read = int96_reader->ReadBatch(1, nullptr, nullptr, &value, &values_read);
- // Ensure only one value is read
- assert(rows_read == 1);
- // There are no NULL values in the rows written
- assert(values_read == 1);
- // Verify the value written
- parquet::Int96 expected_value;
- expected_value.value[0] = col_row_counts[col_id];
- expected_value.value[1] = col_row_counts[col_id] + 1;
- expected_value.value[2] = col_row_counts[col_id] + 2;
- for (int j = 0; j < 3; j++) {
- assert(value.value[j] == expected_value.value[j]);
- }
- col_row_counts[col_id]++;
- }
-
- // Get the Column Reader for the Float column
- col_id++;
- column_reader = row_group_reader->Column(col_id);
- parquet::FloatReader* float_reader =
- static_cast<parquet::FloatReader*>(column_reader.get());
- // Read all the rows in the column
- while (float_reader->HasNext()) {
- float value;
- // Read one value at a time. The number of rows read is returned. values_read
- // contains the number of non-null rows
- rows_read = float_reader->ReadBatch(1, nullptr, nullptr, &value, &values_read);
- // Ensure only one value is read
- assert(rows_read == 1);
- // There are no NULL values in the rows written
- assert(values_read == 1);
- // Verify the value written
- float expected_value = static_cast<float>(col_row_counts[col_id]) * 1.1f;
- assert(value == expected_value);
- col_row_counts[col_id]++;
- }
-
- // Get the Column Reader for the Double column
- col_id++;
- column_reader = row_group_reader->Column(col_id);
- parquet::DoubleReader* double_reader =
- static_cast<parquet::DoubleReader*>(column_reader.get());
- // Read all the rows in the column
- while (double_reader->HasNext()) {
- double value;
- // Read one value at a time. The number of rows read is returned. values_read
- // contains the number of non-null rows
- rows_read = double_reader->ReadBatch(1, nullptr, nullptr, &value, &values_read);
- // Ensure only one value is read
- assert(rows_read == 1);
- // There are no NULL values in the rows written
- assert(values_read == 1);
- // Verify the value written
- double expected_value = col_row_counts[col_id] * 1.1111111;
- assert(value == expected_value);
- col_row_counts[col_id]++;
- }
-
- // Get the Column Reader for the ByteArray column
- col_id++;
- column_reader = row_group_reader->Column(col_id);
- parquet::ByteArrayReader* ba_reader =
- static_cast<parquet::ByteArrayReader*>(column_reader.get());
- // Read all the rows in the column
- while (ba_reader->HasNext()) {
- parquet::ByteArray value;
- // Read one value at a time. The number of rows read is returned. values_read
- // contains the number of non-null rows
- rows_read =
- ba_reader->ReadBatch(1, &definition_level, nullptr, &value, &values_read);
- // Ensure only one value is read
- assert(rows_read == 1);
- // Verify the value written
- char expected_value[FIXED_LENGTH] = "parquet";
- expected_value[7] = static_cast<char>('0' + col_row_counts[col_id] / 100);
- expected_value[8] = static_cast<char>('0' + (col_row_counts[col_id] / 10) % 10);
- expected_value[9] = static_cast<char>('0' + col_row_counts[col_id] % 10);
- if (col_row_counts[col_id] % 2 == 0) { // only alternate values exist
- // There are no NULL values in the rows written
- assert(values_read == 1);
- assert(value.len == FIXED_LENGTH);
- assert(memcmp(value.ptr, &expected_value[0], FIXED_LENGTH) == 0);
- assert(definition_level == 1);
- } else {
- // There are NULL values in the rows written
- assert(values_read == 0);
- assert(definition_level == 0);
- }
- col_row_counts[col_id]++;
- }
-
- // Get the Column Reader for the FixedLengthByteArray column
- col_id++;
- column_reader = row_group_reader->Column(col_id);
- parquet::FixedLenByteArrayReader* flba_reader =
- static_cast<parquet::FixedLenByteArrayReader*>(column_reader.get());
- // Read all the rows in the column
- while (flba_reader->HasNext()) {
- parquet::FixedLenByteArray value;
- // Read one value at a time. The number of rows read is returned. values_read
- // contains the number of non-null rows
- rows_read = flba_reader->ReadBatch(1, nullptr, nullptr, &value, &values_read);
- // Ensure only one value is read
- assert(rows_read == 1);
- // There are no NULL values in the rows written
- assert(values_read == 1);
- // Verify the value written
- char v = static_cast<char>(col_row_counts[col_id]);
- char expected_value[FIXED_LENGTH] = {v, v, v, v, v, v, v, v, v, v};
- assert(memcmp(value.ptr, &expected_value[0], FIXED_LENGTH) == 0);
- col_row_counts[col_id]++;
- }
- }
- } catch (const std::exception& e) {
- std::cerr << "Parquet read error: " << e.what() << std::endl;
- return -1;
- }
-
- std::cout << "Parquet Writing and Reading Complete" << std::endl;
-
- return 0;
-}
diff --git a/examples/low-level-api/reader_writer.h b/examples/low-level-api/reader_writer.h
deleted file mode 100644
index 3fda0cf..0000000
--- a/examples/low-level-api/reader_writer.h
+++ /dev/null
@@ -1,71 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#include <arrow/io/file.h>
-#include <arrow/util/logging.h>
-
-#include <parquet/api/reader.h>
-#include <parquet/api/writer.h>
-
-using parquet::LogicalType;
-using parquet::Repetition;
-using parquet::Type;
-using parquet::schema::GroupNode;
-using parquet::schema::PrimitiveNode;
-
-constexpr int FIXED_LENGTH = 10;
-
-static std::shared_ptr<GroupNode> SetupSchema() {
- parquet::schema::NodeVector fields;
- // Create a primitive node named 'boolean_field' with type:BOOLEAN,
- // repetition:REQUIRED
- fields.push_back(PrimitiveNode::Make("boolean_field", Repetition::REQUIRED,
- Type::BOOLEAN, LogicalType::NONE));
-
- // Create a primitive node named 'int32_field' with type:INT32, repetition:REQUIRED,
- // logical type:TIME_MILLIS
- fields.push_back(PrimitiveNode::Make("int32_field", Repetition::REQUIRED, Type::INT32,
- LogicalType::TIME_MILLIS));
-
- // Create a primitive node named 'int64_field' with type:INT64, repetition:REPEATED
- fields.push_back(PrimitiveNode::Make("int64_field", Repetition::REPEATED, Type::INT64,
- LogicalType::NONE));
-
- fields.push_back(PrimitiveNode::Make("int96_field", Repetition::REQUIRED, Type::INT96,
- LogicalType::NONE));
-
- fields.push_back(PrimitiveNode::Make("float_field", Repetition::REQUIRED, Type::FLOAT,
- LogicalType::NONE));
-
- fields.push_back(PrimitiveNode::Make("double_field", Repetition::REQUIRED, Type::DOUBLE,
- LogicalType::NONE));
-
- // Create a primitive node named 'ba_field' with type:BYTE_ARRAY, repetition:OPTIONAL
- fields.push_back(PrimitiveNode::Make("ba_field", Repetition::OPTIONAL, Type::BYTE_ARRAY,
- LogicalType::NONE));
-
- // Create a primitive node named 'flba_field' with type:FIXED_LEN_BYTE_ARRAY,
- // repetition:REQUIRED, field_length = FIXED_LENGTH
- fields.push_back(PrimitiveNode::Make("flba_field", Repetition::REQUIRED,
- Type::FIXED_LEN_BYTE_ARRAY, LogicalType::NONE,
- FIXED_LENGTH));
-
- // Create a GroupNode named 'schema' using the primitive nodes defined above
- // This GroupNode is the root node of the schema tree
- return std::static_pointer_cast<GroupNode>(
- GroupNode::Make("schema", Repetition::REQUIRED, fields));
-}
diff --git a/examples/parquet-arrow/CMakeLists.txt b/examples/parquet-arrow/CMakeLists.txt
deleted file mode 100644
index 897fcfb..0000000
--- a/examples/parquet-arrow/CMakeLists.txt
+++ /dev/null
@@ -1,78 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-# Require cmake that supports BYPRODUCTS in add_custom_command, ExternalProject_Add [1].
-cmake_minimum_required(VERSION 3.2.0)
-
-project(parquet-arrow-example)
-
-include(ExternalProject)
-include(FindPkgConfig)
-include(GNUInstallDirs)
-
-set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/cmake_modules")
-
-# This ensures that things like gnu++11 get passed correctly
-set(CMAKE_CXX_STANDARD 11)
-
-# We require a C++11 compliant compiler
-set(CMAKE_CXX_STANDARD_REQUIRED ON)
-
-# We want to link dynamically against Arrow and Parquet
-set(PARQUET_BUILD_SHARED ON)
-
-
-# First search the packages in the system. If they are not found, use CMake's
-# ExternalProject mechanism to build them locally.
-find_package(Arrow)
-if (NOT ARROW_FOUND)
- # set compile output directory
- if (NOT CMAKE_BUILD_TYPE)
- set(CMAKE_BUILD_TYPE Debug)
- endif(NOT CMAKE_BUILD_TYPE)
- string (TOLOWER ${CMAKE_BUILD_TYPE} BUILD_SUBDIR_NAME)
- # If build in-source, create the latest symlink. If build out-of-source, which is
- # preferred, simply output the binaries in the build folder
- if (${CMAKE_SOURCE_DIR} STREQUAL "${CMAKE_CURRENT_BINARY_DIR}")
- set(BUILD_OUTPUT_ROOT_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/build/${BUILD_SUBDIR_NAME}")
- # Link build/latest to the current build directory, to avoid developers
- # accidentally running the latest debug build when in fact they're building
- # release builds.
- FILE(MAKE_DIRECTORY ${BUILD_OUTPUT_ROOT_DIRECTORY})
- if (NOT APPLE)
- set(MORE_ARGS "-T")
- endif()
- EXECUTE_PROCESS(COMMAND ln ${MORE_ARGS} -sf ${BUILD_OUTPUT_ROOT_DIRECTORY}
- ${CMAKE_CURRENT_BINARY_DIR}/build/latest)
- else()
- set(BUILD_OUTPUT_ROOT_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/${BUILD_SUBDIR_NAME}")
- endif()
-
- include(ArrowExternalProject)
- set(ARROW_VENDORED 1)
-else()
- set(ARROW_VENDORED 0)
-endif()
-find_package(Parquet)
-
-include_directories(SYSTEM ${ARROW_INCLUDE_DIR} ${PARQUET_INCLUDE_DIR})
-
-add_executable(parquet-arrow-reader-writer src/reader-writer.cc)
-target_link_libraries(parquet-arrow-reader-writer ${PARQUET_SHARED_LIB} ${ARROW_SHARED_LIB})
-if (ARROW_VENDORED)
- add_dependencies(parquet-arrow-reader-writer arrow_ep)
-endif()
diff --git a/examples/parquet-arrow/README.md b/examples/parquet-arrow/README.md
deleted file mode 100644
index e99819f..0000000
--- a/examples/parquet-arrow/README.md
+++ /dev/null
@@ -1,20 +0,0 @@
-<!---
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License. See accompanying LICENSE file.
--->
-
-Using parquet-cpp with the arrow interface
-==========================================
-
-This folder contains an example project that shows how to setup a CMake project
-that consumes `parquet-cpp` as a library as well as how you can use the
-`parquet/arrow` interface to reading and write Apache Parquet files.
diff --git a/examples/parquet-arrow/cmake_modules/ArrowExternalProject.cmake b/examples/parquet-arrow/cmake_modules/ArrowExternalProject.cmake
deleted file mode 120000
index b535f6e..0000000
--- a/examples/parquet-arrow/cmake_modules/ArrowExternalProject.cmake
+++ /dev/null
@@ -1 +0,0 @@
-../../../cmake_modules/ArrowExternalProject.cmake
\ No newline at end of file
diff --git a/examples/parquet-arrow/cmake_modules/FindArrow.cmake b/examples/parquet-arrow/cmake_modules/FindArrow.cmake
deleted file mode 120000
index 6c451ce..0000000
--- a/examples/parquet-arrow/cmake_modules/FindArrow.cmake
+++ /dev/null
@@ -1 +0,0 @@
-../../../cmake_modules/FindArrow.cmake
\ No newline at end of file
diff --git a/examples/parquet-arrow/cmake_modules/FindParquet.cmake b/examples/parquet-arrow/cmake_modules/FindParquet.cmake
deleted file mode 100644
index 8bbe05f..0000000
--- a/examples/parquet-arrow/cmake_modules/FindParquet.cmake
+++ /dev/null
@@ -1,145 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-# - Find PARQUET (parquet/parquet.h, libparquet.a, libparquet.so)
-# This module defines
-# PARQUET_INCLUDE_DIR, directory containing headers
-# PARQUET_LIBS, directory containing parquet libraries
-# PARQUET_STATIC_LIB, path to libparquet.a
-# PARQUET_SHARED_LIB, path to libparquet's shared library
-# PARQUET_SHARED_IMP_LIB, path to libparquet's import library (MSVC only)
-# PARQUET_FOUND, whether parquet has been found
-
-include(FindPkgConfig)
-
-if(NOT "$ENV{PARQUET_HOME}" STREQUAL "")
- set(PARQUET_HOME "$ENV{PARQUET_HOME}")
-endif()
-
-if (MSVC)
- SET(CMAKE_FIND_LIBRARY_SUFFIXES ".lib" ".dll")
-
- if (MSVC AND NOT PARQUET_MSVC_STATIC_LIB_SUFFIX)
- set(PARQUET_MSVC_STATIC_LIB_SUFFIX "_static")
- endif()
-
- find_library(PARQUET_SHARED_LIBRARIES NAMES parquet
- PATHS ${PARQUET_HOME} NO_DEFAULT_PATH
- PATH_SUFFIXES "bin" )
-
- get_filename_component(PARQUET_SHARED_LIBS ${PARQUET_SHARED_LIBRARIES} PATH )
-endif ()
-
-if(PARQUET_HOME)
- set(PARQUET_SEARCH_HEADER_PATHS
- ${PARQUET_HOME}/include
- )
- set(PARQUET_SEARCH_LIB_PATH
- ${PARQUET_HOME}/lib
- )
- find_path(PARQUET_INCLUDE_DIR parquet/api/reader.h PATHS
- ${PARQUET_SEARCH_HEADER_PATHS}
- # make sure we don't accidentally pick up a different version
- NO_DEFAULT_PATH
- )
- find_library(PARQUET_LIBRARIES NAMES parquet
- PATHS ${PARQUET_HOME} NO_DEFAULT_PATH
- PATH_SUFFIXES "lib")
- get_filename_component(PARQUET_LIBS ${PARQUET_LIBRARIES} PATH )
-
- # Try to autodiscover the Parquet ABI version
- get_filename_component(PARQUET_LIB_REALPATH ${PARQUET_LIBRARIES} REALPATH)
- get_filename_component(PARQUET_EXT_REALPATH ${PARQUET_LIB_REALPATH} EXT)
- string(REGEX MATCH ".([0-9]+.[0-9]+.[0-9]+)" HAS_ABI_VERSION ${PARQUET_EXT_REALPATH})
- if (HAS_ABI_VERSION)
- if (APPLE)
- string(REGEX REPLACE ".([0-9]+.[0-9]+.[0-9]+).dylib" "\\1" PARQUET_ABI_VERSION ${PARQUET_EXT_REALPATH})
- else()
- string(REGEX REPLACE ".so.([0-9]+.[0-9]+.[0-9]+)" "\\1" PARQUET_ABI_VERSION ${PARQUET_EXT_REALPATH})
- endif()
- string(REGEX REPLACE "([0-9]+).[0-9]+.[0-9]+" "\\1" PARQUET_SO_VERSION ${PARQUET_ABI_VERSION})
- else()
- set(PARQUET_ABI_VERSION "1.0.0")
- set(PARQUET_SO_VERSION "1")
- endif()
-else()
- pkg_check_modules(PARQUET parquet)
- if (PARQUET_FOUND)
- pkg_get_variable(PARQUET_ABI_VERSION parquet abi_version)
- message(STATUS "Parquet C++ ABI version: ${PARQUET_ABI_VERSION}")
- pkg_get_variable(PARQUET_SO_VERSION parquet so_version)
- message(STATUS "Parquet C++ SO version: ${PARQUET_SO_VERSION}")
- set(PARQUET_INCLUDE_DIR ${PARQUET_INCLUDE_DIRS})
- set(PARQUET_LIBS ${PARQUET_LIBRARY_DIRS})
- set(PARQUET_SEARCH_LIB_PATH ${PARQUET_LIBRARY_DIRS})
- message(STATUS "Searching for parquet libs in: ${PARQUET_SEARCH_LIB_PATH}")
- find_library(PARQUET_LIBRARIES NAMES parquet
- PATHS ${PARQUET_SEARCH_LIB_PATH} NO_DEFAULT_PATH)
- else()
- find_path(PARQUET_INCLUDE_DIR NAMES parquet/api/reader.h )
- find_library(PARQUET_LIBRARIES NAMES parquet)
- get_filename_component(PARQUET_LIBS ${PARQUET_LIBRARIES} PATH )
- endif()
-endif()
-
-if (PARQUET_INCLUDE_DIR AND PARQUET_LIBRARIES)
- set(PARQUET_FOUND TRUE)
- set(PARQUET_LIB_NAME parquet)
- if (MSVC)
- set(PARQUET_STATIC_LIB "${PARQUET_LIBS}/${PARQUET_LIB_NAME}${PARQUET_MSVC_STATIC_LIB_SUFFIX}${CMAKE_STATIC_LIBRARY_SUFFIX}")
- set(PARQUET_SHARED_LIB "${PARQUET_SHARED_LIBS}/${PARQUET_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX}")
- set(PARQUET_SHARED_IMP_LIB "${PARQUET_LIBS}/${PARQUET_LIB_NAME}.lib")
- else()
- set(PARQUET_STATIC_LIB ${PARQUET_LIBS}/${CMAKE_STATIC_LIBRARY_PREFIX}${PARQUET_LIB_NAME}.a)
- set(PARQUET_SHARED_LIB ${PARQUET_LIBS}/${CMAKE_SHARED_LIBRARY_PREFIX}${PARQUET_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX})
- endif()
-else ()
- set(PARQUET_FOUND FALSE)
-endif ()
-
-if (PARQUET_FOUND)
- if (NOT Parquet_FIND_QUIETLY)
- message(STATUS "Found the Parquet library: ${PARQUET_LIBRARIES}")
- endif ()
-else ()
- if (NOT Parquet_FIND_QUIETLY)
- if (NOT PARQUET_FOUND)
- set(PARQUET_ERR_MSG "${PARQUET_ERR_MSG} Could not find the parquet library.")
- endif()
-
- set(PARQUET_ERR_MSG "${PARQUET_ERR_MSG} Looked in ")
- if ( _parquet_roots )
- set(PARQUET_ERR_MSG "${PARQUET_ERR_MSG} in ${_parquet_roots}.")
- else ()
- set(PARQUET_ERR_MSG "${PARQUET_ERR_MSG} system search paths.")
- endif ()
- if (Parquet_FIND_REQUIRED)
- message(FATAL_ERROR "${PARQUET_ERR_MSG}")
- else (Parquet_FIND_REQUIRED)
- message(STATUS "${PARQUET_ERR_MSG}")
- endif (Parquet_FIND_REQUIRED)
- endif ()
-endif ()
-
-mark_as_advanced(
- PARQUET_FOUND
- PARQUET_INCLUDE_DIR
- PARQUET_LIBS
- PARQUET_LIBRARIES
- PARQUET_STATIC_LIB
- PARQUET_SHARED_LIB
-)
diff --git a/examples/parquet-arrow/src/reader-writer.cc b/examples/parquet-arrow/src/reader-writer.cc
deleted file mode 100644
index f333cab..0000000
--- a/examples/parquet-arrow/src/reader-writer.cc
+++ /dev/null
@@ -1,134 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#include <arrow/api.h>
-#include <arrow/io/api.h>
-#include <parquet/arrow/reader.h>
-#include <parquet/arrow/writer.h>
-#include <parquet/exception.h>
-
-// #0 Build dummy data to pass around
-// To have some input data, we first create an Arrow Table that holds
-// some data.
-std::shared_ptr<arrow::Table> generate_table() {
- arrow::Int64Builder i64builder;
- PARQUET_THROW_NOT_OK(i64builder.Append({1, 2, 3, 4, 5}));
- std::shared_ptr<arrow::Array> i64array;
- PARQUET_THROW_NOT_OK(i64builder.Finish(&i64array));
-
- arrow::StringBuilder strbuilder;
- PARQUET_THROW_NOT_OK(strbuilder.Append("some"));
- PARQUET_THROW_NOT_OK(strbuilder.Append("string"));
- PARQUET_THROW_NOT_OK(strbuilder.Append("content"));
- PARQUET_THROW_NOT_OK(strbuilder.Append("in"));
- PARQUET_THROW_NOT_OK(strbuilder.Append("rows"));
- std::shared_ptr<arrow::Array> strarray;
- PARQUET_THROW_NOT_OK(strbuilder.Finish(&strarray));
-
- std::shared_ptr<arrow::Schema> schema = arrow::schema(
- {arrow::field("int", arrow::int64()), arrow::field("str", arrow::utf8())});
-
- return arrow::Table::Make(schema, {i64array, strarray});
-}
-
-// #1 Write out the data as a Parquet file
-void write_parquet_file(const arrow::Table& table) {
- std::shared_ptr<arrow::io::FileOutputStream> outfile;
- PARQUET_THROW_NOT_OK(
- arrow::io::FileOutputStream::Open("parquet-arrow-example.parquet", &outfile));
- // The last argument to the function call is the size of the RowGroup in
- // the parquet file. Normally you would choose this to be rather large but
- // for the example, we use a small value to have multiple RowGroups.
- PARQUET_THROW_NOT_OK(
- parquet::arrow::WriteTable(table, arrow::default_memory_pool(), outfile, 3));
-}
-
-// #2: Fully read in the file
-void read_whole_file() {
- std::cout << "Reading parquet-arrow-example.parquet at once" << std::endl;
- std::shared_ptr<arrow::io::ReadableFile> infile;
- PARQUET_THROW_NOT_OK(arrow::io::ReadableFile::Open(
- "parquet-arrow-example.parquet", arrow::default_memory_pool(), &infile));
-
- std::unique_ptr<parquet::arrow::FileReader> reader;
- PARQUET_THROW_NOT_OK(
- parquet::arrow::OpenFile(infile, arrow::default_memory_pool(), &reader));
- std::shared_ptr<arrow::Table> table;
- PARQUET_THROW_NOT_OK(reader->ReadTable(&table));
- std::cout << "Loaded " << table->num_rows() << " rows in " << table->num_columns()
- << " columns." << std::endl;
-}
-
-// #3: Read only a single RowGroup of the parquet file
-void read_single_rowgroup() {
- std::cout << "Reading first RowGroup of parquet-arrow-example.parquet" << std::endl;
- std::shared_ptr<arrow::io::ReadableFile> infile;
- PARQUET_THROW_NOT_OK(arrow::io::ReadableFile::Open(
- "parquet-arrow-example.parquet", arrow::default_memory_pool(), &infile));
-
- std::unique_ptr<parquet::arrow::FileReader> reader;
- PARQUET_THROW_NOT_OK(
- parquet::arrow::OpenFile(infile, arrow::default_memory_pool(), &reader));
- std::shared_ptr<arrow::Table> table;
- PARQUET_THROW_NOT_OK(reader->RowGroup(0)->ReadTable(&table));
- std::cout << "Loaded " << table->num_rows() << " rows in " << table->num_columns()
- << " columns." << std::endl;
-}
-
-// #4: Read only a single column of the whole parquet file
-void read_single_column() {
- std::cout << "Reading first column of parquet-arrow-example.parquet" << std::endl;
- std::shared_ptr<arrow::io::ReadableFile> infile;
- PARQUET_THROW_NOT_OK(arrow::io::ReadableFile::Open(
- "parquet-arrow-example.parquet", arrow::default_memory_pool(), &infile));
-
- std::unique_ptr<parquet::arrow::FileReader> reader;
- PARQUET_THROW_NOT_OK(
- parquet::arrow::OpenFile(infile, arrow::default_memory_pool(), &reader));
- std::shared_ptr<arrow::Array> array;
- PARQUET_THROW_NOT_OK(reader->ReadColumn(0, &array));
- PARQUET_THROW_NOT_OK(arrow::PrettyPrint(*array, 4, &std::cout));
- std::cout << std::endl;
-}
-
-// #5: Read only a single column of a RowGroup (this is known as ColumnChunk)
-// from the Parquet file.
-void read_single_column_chunk() {
- std::cout << "Reading first ColumnChunk of the first RowGroup of "
- "parquet-arrow-example.parquet"
- << std::endl;
- std::shared_ptr<arrow::io::ReadableFile> infile;
- PARQUET_THROW_NOT_OK(arrow::io::ReadableFile::Open(
- "parquet-arrow-example.parquet", arrow::default_memory_pool(), &infile));
-
- std::unique_ptr<parquet::arrow::FileReader> reader;
- PARQUET_THROW_NOT_OK(
- parquet::arrow::OpenFile(infile, arrow::default_memory_pool(), &reader));
- std::shared_ptr<arrow::Array> array;
- PARQUET_THROW_NOT_OK(reader->RowGroup(0)->Column(0)->Read(&array));
- PARQUET_THROW_NOT_OK(arrow::PrettyPrint(*array, 4, &std::cout));
- std::cout << std::endl;
-}
-
-int main(int argc, char** argv) {
- std::shared_ptr<arrow::Table> table = generate_table();
- write_parquet_file(*table);
- read_whole_file();
- read_single_rowgroup();
- read_single_column();
- read_single_column_chunk();
-}
diff --git a/src/parquet/.gitignore b/src/parquet/.gitignore
deleted file mode 100644
index 0695270..0000000
--- a/src/parquet/.gitignore
+++ /dev/null
@@ -1,2 +0,0 @@
-parquet_constants.*
-parquet_types.*
\ No newline at end of file
diff --git a/src/parquet/CMakeLists.txt b/src/parquet/CMakeLists.txt
deleted file mode 100644
index 93a242c..0000000
--- a/src/parquet/CMakeLists.txt
+++ /dev/null
@@ -1,72 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-# Headers: top level
-install(FILES
- bloom_filter.h
- column_reader.h
- column_page.h
- column_scanner.h
- column_writer.h
- encoding.h
- exception.h
- file_reader.h
- file_writer.h
- hasher.h
- metadata.h
- murmur3.h
- printer.h
- properties.h
- schema.h
- statistics.h
- types.h
- DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/parquet")
-
-configure_file(parquet_version.h.in
- "${CMAKE_CURRENT_SOURCE_DIR}/parquet_version.h"
- @ONLY)
-
-install(FILES
- "${CMAKE_CURRENT_SOURCE_DIR}/parquet_version.h"
- DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/parquet")
-
-# pkg-config support
-configure_file(parquet.pc.in
- "${CMAKE_CURRENT_BINARY_DIR}/parquet.pc"
- @ONLY)
-
-install(FILES
- "${CMAKE_CURRENT_BINARY_DIR}/parquet.pc"
- DESTINATION "${CMAKE_INSTALL_LIBDIR}/pkgconfig/")
-
-ADD_PARQUET_TEST(bloom_filter-test)
-ADD_PARQUET_TEST(column_reader-test)
-ADD_PARQUET_TEST(column_scanner-test)
-ADD_PARQUET_TEST(column_writer-test)
-ADD_PARQUET_TEST(file-deserialize-test)
-ADD_PARQUET_TEST(file-serialize-test)
-ADD_PARQUET_TEST(properties-test)
-ADD_PARQUET_TEST(statistics-test)
-ADD_PARQUET_TEST(encoding-test)
-ADD_PARQUET_TEST(metadata-test)
-ADD_PARQUET_TEST(public-api-test)
-ADD_PARQUET_TEST(types-test)
-ADD_PARQUET_TEST(reader-test)
-ADD_PARQUET_TEST(schema-test)
-
-ADD_PARQUET_BENCHMARK(column-io-benchmark)
-ADD_PARQUET_BENCHMARK(encoding-benchmark)
diff --git a/src/parquet/README b/src/parquet/README
deleted file mode 100644
index fc16a46..0000000
--- a/src/parquet/README
+++ /dev/null
@@ -1,10 +0,0 @@
-The CompatibilityTest of bloom_filter-test.cc is used to test cross compatibility of
-Bloom filters between parquet-mr and parquet-cpp. It reads the Bloom filter binary
-generated by the Bloom filter class in the parquet-mr project and tests whether the
-values inserted before could be filtered or not.
-
-The Bloom filter binary is generated by three steps from Parquet-mr:
-Step 1: Construct a Bloom filter with 1024 bytes of bitset.
-Step 2: Insert hashes of "hello", "parquet", "bloom", "filter" strings to Bloom filter
-by calling hash and insert APIs.
-Step 3: Call writeTo API to write to File.
diff --git a/src/parquet/api/CMakeLists.txt b/src/parquet/api/CMakeLists.txt
deleted file mode 100644
index 79fc716..0000000
--- a/src/parquet/api/CMakeLists.txt
+++ /dev/null
@@ -1,24 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-# Headers: public api
-install(FILES
- io.h
- reader.h
- writer.h
- schema.h
- DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/parquet/api")
diff --git a/src/parquet/api/io.h b/src/parquet/api/io.h
deleted file mode 100644
index 96d3bc0..0000000
--- a/src/parquet/api/io.h
+++ /dev/null
@@ -1,24 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#ifndef PARQUET_API_IO_H
-#define PARQUET_API_IO_H
-
-#include "parquet/exception.h"
-#include "parquet/util/memory.h"
-
-#endif // PARQUET_API_IO_H
diff --git a/src/parquet/api/reader.h b/src/parquet/api/reader.h
deleted file mode 100644
index 505654f..0000000
--- a/src/parquet/api/reader.h
+++ /dev/null
@@ -1,35 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#ifndef PARQUET_API_READER_H
-#define PARQUET_API_READER_H
-
-// Column reader API
-#include "parquet/column_reader.h"
-#include "parquet/column_scanner.h"
-#include "parquet/exception.h"
-#include "parquet/file_reader.h"
-#include "parquet/metadata.h"
-#include "parquet/printer.h"
-
-// Schemas
-#include "parquet/api/schema.h"
-
-// IO
-#include "parquet/api/io.h"
-
-#endif // PARQUET_API_READER_H
diff --git a/src/parquet/api/schema.h b/src/parquet/api/schema.h
deleted file mode 100644
index 2e6c3b3..0000000
--- a/src/parquet/api/schema.h
+++ /dev/null
@@ -1,24 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#ifndef PARQUET_API_SCHEMA_H
-#define PARQUET_API_SCHEMA_H
-
-// Schemas
-#include "parquet/schema.h"
-
-#endif // PARQUET_API_SCHEMA_H
diff --git a/src/parquet/api/writer.h b/src/parquet/api/writer.h
deleted file mode 100644
index 3b4e42f..0000000
--- a/src/parquet/api/writer.h
+++ /dev/null
@@ -1,27 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#ifndef PARQUET_API_WRITER_H
-#define PARQUET_API_WRITER_H
-
-#include "parquet/api/io.h"
-#include "parquet/api/schema.h"
-#include "parquet/column_writer.h"
-#include "parquet/exception.h"
-#include "parquet/file_writer.h"
-
-#endif // PARQUET_API_WRITER_H
diff --git a/src/parquet/arrow/CMakeLists.txt b/src/parquet/arrow/CMakeLists.txt
deleted file mode 100644
index 616555f..0000000
--- a/src/parquet/arrow/CMakeLists.txt
+++ /dev/null
@@ -1,30 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-ADD_PARQUET_TEST(arrow-schema-test)
-ADD_PARQUET_TEST(arrow-reader-writer-test)
-
-if(PARQUET_BUILD_BENCHMARKS)
- ADD_PARQUET_BENCHMARK(arrow-reader-writer-benchmark)
-endif()
-
-# Headers: top level
-install(FILES
- reader.h
- schema.h
- writer.h
- DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/parquet/arrow")
diff --git a/src/parquet/arrow/arrow-reader-writer-benchmark.cc b/src/parquet/arrow/arrow-reader-writer-benchmark.cc
deleted file mode 100644
index 41cb88d..0000000
--- a/src/parquet/arrow/arrow-reader-writer-benchmark.cc
+++ /dev/null
@@ -1,263 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#include "benchmark/benchmark.h"
-
-#include <iostream>
-
-#include "parquet/arrow/reader.h"
-#include "parquet/arrow/writer.h"
-#include "parquet/column_reader.h"
-#include "parquet/column_writer.h"
-#include "parquet/file_reader.h"
-#include "parquet/file_writer.h"
-#include "parquet/util/memory.h"
-
-#include "arrow/api.h"
-
-using arrow::BooleanBuilder;
-using arrow::NumericBuilder;
-
-#define EXIT_NOT_OK(s) \
- do { \
- ::arrow::Status _s = (s); \
- if (ARROW_PREDICT_FALSE(!_s.ok())) { \
- std::cout << "Exiting: " << _s.ToString() << std::endl; \
- exit(EXIT_FAILURE); \
- } \
- } while (0)
-
-namespace parquet {
-
-using arrow::FileReader;
-using arrow::WriteTable;
-using schema::PrimitiveNode;
-
-namespace benchmark {
-
-// This should result in multiple pages for most primitive types
-constexpr int64_t BENCHMARK_SIZE = 10 * 1024 * 1024;
-
-template <typename ParquetType>
-struct benchmark_traits {};
-
-template <>
-struct benchmark_traits<Int32Type> {
- using arrow_type = ::arrow::Int32Type;
-};
-
-template <>
-struct benchmark_traits<Int64Type> {
- using arrow_type = ::arrow::Int64Type;
-};
-
-template <>
-struct benchmark_traits<DoubleType> {
- using arrow_type = ::arrow::DoubleType;
-};
-
-template <>
-struct benchmark_traits<BooleanType> {
- using arrow_type = ::arrow::BooleanType;
-};
-
-template <typename ParquetType>
-using ArrowType = typename benchmark_traits<ParquetType>::arrow_type;
-
-template <typename ParquetType>
-std::shared_ptr<ColumnDescriptor> MakeSchema(Repetition::type repetition) {
- auto node = PrimitiveNode::Make("int64", repetition, ParquetType::type_num);
- return std::make_shared<ColumnDescriptor>(node, repetition != Repetition::REQUIRED,
- repetition == Repetition::REPEATED);
-}
-
-template <bool nullable, typename ParquetType>
-void SetBytesProcessed(::benchmark::State& state) {
- int64_t bytes_processed =
- state.iterations() * BENCHMARK_SIZE * sizeof(typename ParquetType::c_type);
- if (nullable) {
- bytes_processed += state.iterations() * BENCHMARK_SIZE * sizeof(int16_t);
- }
- state.SetBytesProcessed(bytes_processed);
-}
-
-template <typename ParquetType>
-std::shared_ptr<::arrow::Table> TableFromVector(
- const std::vector<typename ParquetType::c_type>& vec, bool nullable) {
- ::arrow::TypePtr type = std::make_shared<ArrowType<ParquetType>>();
- NumericBuilder<ArrowType<ParquetType>> builder;
- if (nullable) {
- std::vector<uint8_t> valid_bytes(BENCHMARK_SIZE, 0);
- int n = {0};
- std::generate(valid_bytes.begin(), valid_bytes.end(), [&n] { return n++ % 2; });
- EXIT_NOT_OK(builder.AppendValues(vec.data(), vec.size(), valid_bytes.data()));
- } else {
- EXIT_NOT_OK(builder.AppendValues(vec.data(), vec.size(), nullptr));
- }
- std::shared_ptr<::arrow::Array> array;
- EXIT_NOT_OK(builder.Finish(&array));
-
- auto field = ::arrow::field("column", type, nullable);
- auto schema = ::arrow::schema({field});
- auto column = std::make_shared<::arrow::Column>(field, array);
- return ::arrow::Table::Make(schema, {column});
-}
-
-template <>
-std::shared_ptr<::arrow::Table> TableFromVector<BooleanType>(const std::vector<bool>& vec,
- bool nullable) {
- BooleanBuilder builder;
- if (nullable) {
- std::vector<bool> valid_bytes(BENCHMARK_SIZE, 0);
- int n = {0};
- std::generate(valid_bytes.begin(), valid_bytes.end(),
- [&n] { return (n++ % 2) != 0; });
- EXIT_NOT_OK(builder.AppendValues(vec, valid_bytes));
- } else {
- EXIT_NOT_OK(builder.AppendValues(vec));
- }
- std::shared_ptr<::arrow::Array> array;
- EXIT_NOT_OK(builder.Finish(&array));
-
- auto field = ::arrow::field("column", ::arrow::boolean(), nullable);
- auto schema = std::make_shared<::arrow::Schema>(
- std::vector<std::shared_ptr<::arrow::Field>>({field}));
- auto column = std::make_shared<::arrow::Column>(field, array);
- return ::arrow::Table::Make(schema, {column});
-}
-
-template <bool nullable, typename ParquetType>
-static void BM_WriteColumn(::benchmark::State& state) {
- std::vector<typename ParquetType::c_type> values(BENCHMARK_SIZE, 128);
- std::shared_ptr<::arrow::Table> table = TableFromVector<ParquetType>(values, nullable);
-
- while (state.KeepRunning()) {
- auto output = std::make_shared<InMemoryOutputStream>();
- EXIT_NOT_OK(
- WriteTable(*table, ::arrow::default_memory_pool(), output, BENCHMARK_SIZE));
- }
- SetBytesProcessed<nullable, ParquetType>(state);
-}
-
-BENCHMARK_TEMPLATE2(BM_WriteColumn, false, Int32Type);
-BENCHMARK_TEMPLATE2(BM_WriteColumn, true, Int32Type);
-
-BENCHMARK_TEMPLATE2(BM_WriteColumn, false, Int64Type);
-BENCHMARK_TEMPLATE2(BM_WriteColumn, true, Int64Type);
-
-BENCHMARK_TEMPLATE2(BM_WriteColumn, false, DoubleType);
-BENCHMARK_TEMPLATE2(BM_WriteColumn, true, DoubleType);
-
-BENCHMARK_TEMPLATE2(BM_WriteColumn, false, BooleanType);
-BENCHMARK_TEMPLATE2(BM_WriteColumn, true, BooleanType);
-
-template <bool nullable, typename ParquetType>
-static void BM_ReadColumn(::benchmark::State& state) {
- std::vector<typename ParquetType::c_type> values(BENCHMARK_SIZE, 128);
- std::shared_ptr<::arrow::Table> table = TableFromVector<ParquetType>(values, nullable);
- auto output = std::make_shared<InMemoryOutputStream>();
- EXIT_NOT_OK(WriteTable(*table, ::arrow::default_memory_pool(), output, BENCHMARK_SIZE));
- std::shared_ptr<Buffer> buffer = output->GetBuffer();
-
- while (state.KeepRunning()) {
- auto reader =
- ParquetFileReader::Open(std::make_shared<::arrow::io::BufferReader>(buffer));
- FileReader filereader(::arrow::default_memory_pool(), std::move(reader));
- std::shared_ptr<::arrow::Table> table;
- EXIT_NOT_OK(filereader.ReadTable(&table));
- }
- SetBytesProcessed<nullable, ParquetType>(state);
-}
-
-BENCHMARK_TEMPLATE2(BM_ReadColumn, false, Int32Type);
-BENCHMARK_TEMPLATE2(BM_ReadColumn, true, Int32Type);
-
-BENCHMARK_TEMPLATE2(BM_ReadColumn, false, Int64Type);
-BENCHMARK_TEMPLATE2(BM_ReadColumn, true, Int64Type);
-
-BENCHMARK_TEMPLATE2(BM_ReadColumn, false, DoubleType);
-BENCHMARK_TEMPLATE2(BM_ReadColumn, true, DoubleType);
-
-BENCHMARK_TEMPLATE2(BM_ReadColumn, false, BooleanType);
-BENCHMARK_TEMPLATE2(BM_ReadColumn, true, BooleanType);
-
-static void BM_ReadIndividualRowGroups(::benchmark::State& state) {
- std::vector<int64_t> values(BENCHMARK_SIZE, 128);
- std::shared_ptr<::arrow::Table> table = TableFromVector<Int64Type>(values, true);
- auto output = std::make_shared<InMemoryOutputStream>();
- // This writes 10 RowGroups
- EXIT_NOT_OK(
- WriteTable(*table, ::arrow::default_memory_pool(), output, BENCHMARK_SIZE / 10));
- std::shared_ptr<Buffer> buffer = output->GetBuffer();
-
- while (state.KeepRunning()) {
- auto reader =
- ParquetFileReader::Open(std::make_shared<::arrow::io::BufferReader>(buffer));
- FileReader filereader(::arrow::default_memory_pool(), std::move(reader));
-
- std::vector<std::shared_ptr<::arrow::Table>> tables;
- for (int i = 0; i < filereader.num_row_groups(); i++) {
- // Only read the even numbered RowGroups
- if ((i % 2) == 0) {
- std::shared_ptr<::arrow::Table> table;
- EXIT_NOT_OK(filereader.RowGroup(i)->ReadTable(&table));
- tables.push_back(table);
- }
- }
-
- std::shared_ptr<::arrow::Table> final_table;
- EXIT_NOT_OK(ConcatenateTables(tables, &final_table));
- }
- SetBytesProcessed<true, Int64Type>(state);
-}
-
-BENCHMARK(BM_ReadIndividualRowGroups);
-
-static void BM_ReadMultipleRowGroups(::benchmark::State& state) {
- std::vector<int64_t> values(BENCHMARK_SIZE, 128);
- std::shared_ptr<::arrow::Table> table = TableFromVector<Int64Type>(values, true);
- auto output = std::make_shared<InMemoryOutputStream>();
- // This writes 10 RowGroups
- EXIT_NOT_OK(
- WriteTable(*table, ::arrow::default_memory_pool(), output, BENCHMARK_SIZE / 10));
- std::shared_ptr<Buffer> buffer = output->GetBuffer();
-
- while (state.KeepRunning()) {
- auto reader =
- ParquetFileReader::Open(std::make_shared<::arrow::io::BufferReader>(buffer));
- FileReader filereader(::arrow::default_memory_pool(), std::move(reader));
-
- std::vector<std::shared_ptr<::arrow::Table>> tables;
- std::vector<int> rgs;
- for (int i = 0; i < filereader.num_row_groups(); i++) {
- // Only read the even numbered RowGroups
- if ((i % 2) == 0) {
- rgs.push_back(i);
- }
- }
-
- std::shared_ptr<::arrow::Table> table;
- EXIT_NOT_OK(filereader.ReadRowGroups(rgs, &table));
- }
- SetBytesProcessed<true, Int64Type>(state);
-}
-
-BENCHMARK(BM_ReadMultipleRowGroups);
-
-} // namespace benchmark
-
-} // namespace parquet
diff --git a/src/parquet/arrow/arrow-reader-writer-test.cc b/src/parquet/arrow/arrow-reader-writer-test.cc
deleted file mode 100644
index 5f4e123..0000000
--- a/src/parquet/arrow/arrow-reader-writer-test.cc
+++ /dev/null
@@ -1,2376 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#ifdef _MSC_VER
-#pragma warning(push)
-// Disable forcing value to bool warnings
-#pragma warning(disable : 4800)
-#endif
-
-#include "gtest/gtest.h"
-
-#include <arrow/compute/api.h>
-#include <cstdint>
-#include <functional>
-#include <sstream>
-#include <vector>
-
-#include "parquet/api/reader.h"
-#include "parquet/api/writer.h"
-
-#include "parquet/arrow/reader.h"
-#include "parquet/arrow/schema.h"
-#include "parquet/arrow/test-util.h"
-#include "parquet/arrow/writer.h"
-
-#include "parquet/file_writer.h"
-
-#include "parquet/util/test-common.h"
-
-#include "arrow/api.h"
-#include "arrow/test-util.h"
-#include "arrow/type_traits.h"
-#include "arrow/util/decimal.h"
-
-using arrow::Array;
-using arrow::ArrayVisitor;
-using arrow::Buffer;
-using arrow::ChunkedArray;
-using arrow::Column;
-using arrow::DataType;
-using arrow::default_memory_pool;
-using arrow::ListArray;
-using arrow::PrimitiveArray;
-using arrow::ResizableBuffer;
-using arrow::Status;
-using arrow::Table;
-using arrow::TimeUnit;
-using arrow::compute::Datum;
-using arrow::compute::DictionaryEncode;
-using arrow::compute::FunctionContext;
-using arrow::io::BufferReader;
-
-using arrow::randint;
-using arrow::random_is_valid;
-
-using ArrowId = ::arrow::Type;
-using ParquetType = parquet::Type;
-using parquet::arrow::FromParquetSchema;
-using parquet::schema::GroupNode;
-using parquet::schema::NodePtr;
-using parquet::schema::PrimitiveNode;
-
-using ColumnVector = std::vector<std::shared_ptr<arrow::Column>>;
-
-namespace parquet {
-namespace arrow {
-
-static constexpr int SMALL_SIZE = 100;
-#ifdef PARQUET_VALGRIND
-static constexpr int LARGE_SIZE = 1000;
-#else
-static constexpr int LARGE_SIZE = 10000;
-#endif
-
-static constexpr uint32_t kDefaultSeed = 0;
-
-LogicalType::type get_logical_type(const ::DataType& type) {
- switch (type.id()) {
- case ArrowId::UINT8:
- return LogicalType::UINT_8;
- case ArrowId::INT8:
- return LogicalType::INT_8;
- case ArrowId::UINT16:
- return LogicalType::UINT_16;
- case ArrowId::INT16:
- return LogicalType::INT_16;
- case ArrowId::UINT32:
- return LogicalType::UINT_32;
- case ArrowId::INT32:
- return LogicalType::INT_32;
- case ArrowId::UINT64:
- return LogicalType::UINT_64;
- case ArrowId::INT64:
- return LogicalType::INT_64;
- case ArrowId::STRING:
- return LogicalType::UTF8;
- case ArrowId::DATE32:
- return LogicalType::DATE;
- case ArrowId::DATE64:
- return LogicalType::DATE;
- case ArrowId::TIMESTAMP: {
- const auto& ts_type = static_cast<const ::arrow::TimestampType&>(type);
- switch (ts_type.unit()) {
- case TimeUnit::MILLI:
- return LogicalType::TIMESTAMP_MILLIS;
- case TimeUnit::MICRO:
- return LogicalType::TIMESTAMP_MICROS;
- default:
- DCHECK(false) << "Only MILLI and MICRO units supported for Arrow timestamps "
- "with Parquet.";
- }
- }
- case ArrowId::TIME32:
- return LogicalType::TIME_MILLIS;
- case ArrowId::TIME64:
- return LogicalType::TIME_MICROS;
- case ArrowId::DICTIONARY: {
- const ::arrow::DictionaryType& dict_type =
- static_cast<const ::arrow::DictionaryType&>(type);
- return get_logical_type(*dict_type.dictionary()->type());
- }
- case ArrowId::DECIMAL:
- return LogicalType::DECIMAL;
- default:
- break;
- }
- return LogicalType::NONE;
-}
-
-ParquetType::type get_physical_type(const ::DataType& type) {
- switch (type.id()) {
- case ArrowId::BOOL:
- return ParquetType::BOOLEAN;
- case ArrowId::UINT8:
- case ArrowId::INT8:
- case ArrowId::UINT16:
- case ArrowId::INT16:
- case ArrowId::UINT32:
- case ArrowId::INT32:
- return ParquetType::INT32;
- case ArrowId::UINT64:
- case ArrowId::INT64:
- return ParquetType::INT64;
- case ArrowId::FLOAT:
- return ParquetType::FLOAT;
- case ArrowId::DOUBLE:
- return ParquetType::DOUBLE;
- case ArrowId::BINARY:
- return ParquetType::BYTE_ARRAY;
- case ArrowId::STRING:
- return ParquetType::BYTE_ARRAY;
- case ArrowId::FIXED_SIZE_BINARY:
- case ArrowId::DECIMAL:
- return ParquetType::FIXED_LEN_BYTE_ARRAY;
- case ArrowId::DATE32:
- return ParquetType::INT32;
- case ArrowId::DATE64:
- // Convert to date32 internally
- return ParquetType::INT32;
- case ArrowId::TIME32:
- return ParquetType::INT32;
- case ArrowId::TIME64:
- return ParquetType::INT64;
- case ArrowId::TIMESTAMP:
- return ParquetType::INT64;
- case ArrowId::DICTIONARY: {
- const ::arrow::DictionaryType& dict_type =
- static_cast<const ::arrow::DictionaryType&>(type);
- return get_physical_type(*dict_type.dictionary()->type());
- }
- default:
- break;
- }
- DCHECK(false) << "cannot reach this code";
- return ParquetType::INT32;
-}
-
-template <typename TestType>
-struct test_traits {};
-
-template <>
-struct test_traits<::arrow::BooleanType> {
- static constexpr ParquetType::type parquet_enum = ParquetType::BOOLEAN;
- static uint8_t const value;
-};
-
-const uint8_t test_traits<::arrow::BooleanType>::value(1);
-
-template <>
-struct test_traits<::arrow::UInt8Type> {
- static constexpr ParquetType::type parquet_enum = ParquetType::INT32;
- static uint8_t const value;
-};
-
-const uint8_t test_traits<::arrow::UInt8Type>::value(64);
-
-template <>
-struct test_traits<::arrow::Int8Type> {
- static constexpr ParquetType::type parquet_enum = ParquetType::INT32;
- static int8_t const value;
-};
-
-const int8_t test_traits<::arrow::Int8Type>::value(-64);
-
-template <>
-struct test_traits<::arrow::UInt16Type> {
- static constexpr ParquetType::type parquet_enum = ParquetType::INT32;
- static uint16_t const value;
-};
-
-const uint16_t test_traits<::arrow::UInt16Type>::value(1024);
-
-template <>
-struct test_traits<::arrow::Int16Type> {
- static constexpr ParquetType::type parquet_enum = ParquetType::INT32;
- static int16_t const value;
-};
-
-const int16_t test_traits<::arrow::Int16Type>::value(-1024);
-
-template <>
-struct test_traits<::arrow::UInt32Type> {
- static constexpr ParquetType::type parquet_enum = ParquetType::INT32;
- static uint32_t const value;
-};
-
-const uint32_t test_traits<::arrow::UInt32Type>::value(1024);
-
-template <>
-struct test_traits<::arrow::Int32Type> {
- static constexpr ParquetType::type parquet_enum = ParquetType::INT32;
- static int32_t const value;
-};
-
-const int32_t test_traits<::arrow::Int32Type>::value(-1024);
-
-template <>
-struct test_traits<::arrow::UInt64Type> {
- static constexpr ParquetType::type parquet_enum = ParquetType::INT64;
- static uint64_t const value;
-};
-
-const uint64_t test_traits<::arrow::UInt64Type>::value(1024);
-
-template <>
-struct test_traits<::arrow::Int64Type> {
- static constexpr ParquetType::type parquet_enum = ParquetType::INT64;
- static int64_t const value;
-};
-
-const int64_t test_traits<::arrow::Int64Type>::value(-1024);
-
-template <>
-struct test_traits<::arrow::TimestampType> {
- static constexpr ParquetType::type parquet_enum = ParquetType::INT64;
- static int64_t const value;
-};
-
-const int64_t test_traits<::arrow::TimestampType>::value(14695634030000);
-
-template <>
-struct test_traits<::arrow::Date32Type> {
- static constexpr ParquetType::type parquet_enum = ParquetType::INT32;
- static int32_t const value;
-};
-
-const int32_t test_traits<::arrow::Date32Type>::value(170000);
-
-template <>
-struct test_traits<::arrow::FloatType> {
- static constexpr ParquetType::type parquet_enum = ParquetType::FLOAT;
- static float const value;
-};
-
-const float test_traits<::arrow::FloatType>::value(2.1f);
-
-template <>
-struct test_traits<::arrow::DoubleType> {
- static constexpr ParquetType::type parquet_enum = ParquetType::DOUBLE;
- static double const value;
-};
-
-const double test_traits<::arrow::DoubleType>::value(4.2);
-
-template <>
-struct test_traits<::arrow::StringType> {
- static constexpr ParquetType::type parquet_enum = ParquetType::BYTE_ARRAY;
- static std::string const value;
-};
-
-template <>
-struct test_traits<::arrow::BinaryType> {
- static constexpr ParquetType::type parquet_enum = ParquetType::BYTE_ARRAY;
- static std::string const value;
-};
-
-template <>
-struct test_traits<::arrow::FixedSizeBinaryType> {
- static constexpr ParquetType::type parquet_enum = ParquetType::FIXED_LEN_BYTE_ARRAY;
- static std::string const value;
-};
-
-const std::string test_traits<::arrow::StringType>::value("Test"); // NOLINT
-const std::string test_traits<::arrow::BinaryType>::value("\x00\x01\x02\x03"); // NOLINT
-const std::string test_traits<::arrow::FixedSizeBinaryType>::value("Fixed"); // NOLINT
-
-template <typename T>
-using ParquetDataType = DataType<test_traits<T>::parquet_enum>;
-
-template <typename T>
-using ParquetWriter = TypedColumnWriter<ParquetDataType<T>>;
-
-void WriteTableToBuffer(const std::shared_ptr<Table>& table, int64_t row_group_size,
- const std::shared_ptr<ArrowWriterProperties>& arrow_properties,
- std::shared_ptr<Buffer>* out) {
- auto sink = std::make_shared<InMemoryOutputStream>();
-
- ASSERT_OK_NO_THROW(WriteTable(*table, ::arrow::default_memory_pool(), sink,
- row_group_size, default_writer_properties(),
- arrow_properties));
- *out = sink->GetBuffer();
-}
-
-namespace internal {
-
-void AssertArraysEqual(const Array& expected, const Array& actual) {
- if (!actual.Equals(expected)) {
- std::stringstream pp_result;
- std::stringstream pp_expected;
-
- EXPECT_OK(::arrow::PrettyPrint(actual, 0, &pp_result));
- EXPECT_OK(::arrow::PrettyPrint(expected, 0, &pp_expected));
- FAIL() << "Got: \n" << pp_result.str() << "\nExpected: \n" << pp_expected.str();
- }
-}
-
-} // namespace internal
-
-void AssertChunkedEqual(const ChunkedArray& expected, const ChunkedArray& actual) {
- ASSERT_EQ(expected.num_chunks(), actual.num_chunks()) << "# chunks unequal";
- if (!actual.Equals(expected)) {
- std::stringstream pp_result;
- std::stringstream pp_expected;
-
- for (int i = 0; i < actual.num_chunks(); ++i) {
- auto c1 = actual.chunk(i);
- auto c2 = expected.chunk(i);
- if (!c1->Equals(*c2)) {
- EXPECT_OK(::arrow::PrettyPrint(*c1, 0, &pp_result));
- EXPECT_OK(::arrow::PrettyPrint(*c2, 0, &pp_expected));
- FAIL() << "Chunk " << i << " Got: " << pp_result.str()
- << "\nExpected: " << pp_expected.str();
- }
- }
- }
-}
-
-void PrintColumn(const Column& col, std::stringstream* ss) {
- const ChunkedArray& carr = *col.data();
- for (int i = 0; i < carr.num_chunks(); ++i) {
- auto c1 = carr.chunk(i);
- *ss << "Chunk " << i << std::endl;
- EXPECT_OK(::arrow::PrettyPrint(*c1, 0, ss));
- *ss << std::endl;
- }
-}
-
-void DoSimpleRoundtrip(const std::shared_ptr<Table>& table, bool use_threads,
- int64_t row_group_size, const std::vector<int>& column_subset,
- std::shared_ptr<Table>* out,
- const std::shared_ptr<ArrowWriterProperties>& arrow_properties =
- default_arrow_writer_properties()) {
- std::shared_ptr<Buffer> buffer;
- ASSERT_NO_FATAL_FAILURE(
- WriteTableToBuffer(table, row_group_size, arrow_properties, &buffer));
-
- std::unique_ptr<FileReader> reader;
- ASSERT_OK_NO_THROW(OpenFile(std::make_shared<BufferReader>(buffer),
- ::arrow::default_memory_pool(),
- ::parquet::default_reader_properties(), nullptr, &reader));
-
- reader->set_use_threads(use_threads);
-
- if (column_subset.size() > 0) {
- ASSERT_OK_NO_THROW(reader->ReadTable(column_subset, out));
- } else {
- // Read everything
- ASSERT_OK_NO_THROW(reader->ReadTable(out));
- }
-}
-
-void CheckSimpleRoundtrip(const std::shared_ptr<Table>& table, int64_t row_group_size,
- const std::shared_ptr<ArrowWriterProperties>& arrow_properties =
- default_arrow_writer_properties()) {
- std::shared_ptr<Table> result;
- DoSimpleRoundtrip(table, false /* use_threads */, row_group_size, {}, &result,
- arrow_properties);
- ASSERT_NO_FATAL_FAILURE(::arrow::AssertTablesEqual(*table, *result, false));
-}
-
-static std::shared_ptr<GroupNode> MakeSimpleSchema(const ::DataType& type,
- Repetition::type repetition) {
- int32_t byte_width = -1;
- int32_t precision = -1;
- int32_t scale = -1;
-
- switch (type.id()) {
- case ::arrow::Type::DICTIONARY: {
- const auto& dict_type = static_cast<const ::arrow::DictionaryType&>(type);
- const ::DataType& values_type = *dict_type.dictionary()->type();
- switch (values_type.id()) {
- case ::arrow::Type::FIXED_SIZE_BINARY:
- byte_width =
- static_cast<const ::arrow::FixedSizeBinaryType&>(values_type).byte_width();
- break;
- case ::arrow::Type::DECIMAL: {
- const auto& decimal_type =
- static_cast<const ::arrow::Decimal128Type&>(values_type);
- precision = decimal_type.precision();
- scale = decimal_type.scale();
- byte_width = DecimalSize(precision);
- } break;
- default:
- break;
- }
- } break;
- case ::arrow::Type::FIXED_SIZE_BINARY:
- byte_width = static_cast<const ::arrow::FixedSizeBinaryType&>(type).byte_width();
- break;
- case ::arrow::Type::DECIMAL: {
- const auto& decimal_type = static_cast<const ::arrow::Decimal128Type&>(type);
- precision = decimal_type.precision();
- scale = decimal_type.scale();
- byte_width = DecimalSize(precision);
- } break;
- default:
- break;
- }
- auto pnode = PrimitiveNode::Make("column1", repetition, get_physical_type(type),
- get_logical_type(type), byte_width, precision, scale);
- NodePtr node_ =
- GroupNode::Make("schema", Repetition::REQUIRED, std::vector<NodePtr>({pnode}));
- return std::static_pointer_cast<GroupNode>(node_);
-}
-
-template <typename TestType>
-class TestParquetIO : public ::testing::Test {
- public:
- virtual void SetUp() {}
-
- std::unique_ptr<ParquetFileWriter> MakeWriter(
- const std::shared_ptr<GroupNode>& schema) {
- sink_ = std::make_shared<InMemoryOutputStream>();
- return ParquetFileWriter::Open(sink_, schema);
- }
-
- void ReaderFromSink(std::unique_ptr<FileReader>* out) {
- std::shared_ptr<Buffer> buffer = sink_->GetBuffer();
- ASSERT_OK_NO_THROW(OpenFile(std::make_shared<BufferReader>(buffer),
- ::arrow::default_memory_pool(),
- ::parquet::default_reader_properties(), nullptr, out));
- }
-
- void ReadSingleColumnFile(std::unique_ptr<FileReader> file_reader,
- std::shared_ptr<Array>* out) {
- std::unique_ptr<ColumnReader> column_reader;
- ASSERT_OK_NO_THROW(file_reader->GetColumn(0, &column_reader));
- ASSERT_NE(nullptr, column_reader.get());
-
- ASSERT_OK(column_reader->NextBatch(SMALL_SIZE, out));
- ASSERT_NE(nullptr, out->get());
- }
-
- void ReadAndCheckSingleColumnFile(const Array& values) {
- std::shared_ptr<Array> out;
-
- std::unique_ptr<FileReader> reader;
- ReaderFromSink(&reader);
- ReadSingleColumnFile(std::move(reader), &out);
-
- internal::AssertArraysEqual(values, *out);
- }
-
- void ReadTableFromFile(std::unique_ptr<FileReader> reader,
- std::shared_ptr<Table>* out) {
- ASSERT_OK_NO_THROW(reader->ReadTable(out));
- auto key_value_metadata =
- reader->parquet_reader()->metadata()->key_value_metadata().get();
- ASSERT_EQ(nullptr, key_value_metadata);
- ASSERT_NE(nullptr, out->get());
- }
-
- void PrepareListTable(int64_t size, bool nullable_lists, bool nullable_elements,
- int64_t null_count, std::shared_ptr<Table>* out) {
- std::shared_ptr<Array> values;
- ASSERT_OK(NullableArray<TestType>(size * size, nullable_elements ? null_count : 0,
- kDefaultSeed, &values));
- // Also test that slice offsets are respected
- values = values->Slice(5, values->length() - 5);
- std::shared_ptr<ListArray> lists;
- ASSERT_OK(MakeListArray(values, size, nullable_lists ? null_count : 0,
- nullable_elements, &lists));
- *out = MakeSimpleTable(lists->Slice(3, size - 6), nullable_lists);
- }
-
- // Prepare table of empty lists, with null values array (ARROW-2744)
- void PrepareEmptyListsTable(int64_t size, std::shared_ptr<Table>* out) {
- std::shared_ptr<Array> lists;
- ASSERT_OK(MakeEmptyListsArray(size, &lists));
- *out = MakeSimpleTable(lists, true /* nullable_lists */);
- }
-
- void PrepareListOfListTable(int64_t size, bool nullable_parent_lists,
- bool nullable_lists, bool nullable_elements,
- int64_t null_count, std::shared_ptr<Table>* out) {
- std::shared_ptr<Array> values;
- ASSERT_OK(NullableArray<TestType>(size * 6, nullable_elements ? null_count : 0,
- kDefaultSeed, &values));
- std::shared_ptr<ListArray> lists;
- ASSERT_OK(MakeListArray(values, size * 3, nullable_lists ? null_count : 0,
- nullable_elements, &lists));
- std::shared_ptr<ListArray> parent_lists;
- ASSERT_OK(MakeListArray(lists, size, nullable_parent_lists ? null_count : 0,
- nullable_lists, &parent_lists));
- *out = MakeSimpleTable(parent_lists, nullable_parent_lists);
- }
-
- void ReadAndCheckSingleColumnTable(const std::shared_ptr<Array>& values) {
- std::shared_ptr<::arrow::Table> out;
- std::unique_ptr<FileReader> reader;
- ReaderFromSink(&reader);
- ReadTableFromFile(std::move(reader), &out);
- ASSERT_EQ(1, out->num_columns());
- ASSERT_EQ(values->length(), out->num_rows());
-
- std::shared_ptr<ChunkedArray> chunked_array = out->column(0)->data();
- ASSERT_EQ(1, chunked_array->num_chunks());
- auto result = chunked_array->chunk(0);
-
- internal::AssertArraysEqual(*values, *result);
- }
-
- void CheckRoundTrip(const std::shared_ptr<Table>& table) {
- CheckSimpleRoundtrip(table, table->num_rows());
- }
-
- template <typename ArrayType>
- void WriteColumn(const std::shared_ptr<GroupNode>& schema,
- const std::shared_ptr<ArrayType>& values) {
- FileWriter writer(::arrow::default_memory_pool(), MakeWriter(schema));
- ASSERT_OK_NO_THROW(writer.NewRowGroup(values->length()));
- ASSERT_OK_NO_THROW(writer.WriteColumnChunk(*values));
- ASSERT_OK_NO_THROW(writer.Close());
- // writer.Close() should be idempotent
- ASSERT_OK_NO_THROW(writer.Close());
- }
-
- std::shared_ptr<InMemoryOutputStream> sink_;
-};
-
-// We have separate tests for UInt32Type as this is currently the only type
-// where a roundtrip does not yield the identical Array structure.
-// There we write an UInt32 Array but receive an Int64 Array as result for
-// Parquet version 1.0.
-
-typedef ::testing::Types<
- ::arrow::BooleanType, ::arrow::UInt8Type, ::arrow::Int8Type, ::arrow::UInt16Type,
- ::arrow::Int16Type, ::arrow::Int32Type, ::arrow::UInt64Type, ::arrow::Int64Type,
- ::arrow::Date32Type, ::arrow::FloatType, ::arrow::DoubleType, ::arrow::StringType,
- ::arrow::BinaryType, ::arrow::FixedSizeBinaryType, DecimalWithPrecisionAndScale<1>,
- DecimalWithPrecisionAndScale<3>, DecimalWithPrecisionAndScale<5>,
- DecimalWithPrecisionAndScale<7>, DecimalWithPrecisionAndScale<10>,
- DecimalWithPrecisionAndScale<12>, DecimalWithPrecisionAndScale<15>,
- DecimalWithPrecisionAndScale<17>, DecimalWithPrecisionAndScale<19>,
- DecimalWithPrecisionAndScale<22>, DecimalWithPrecisionAndScale<23>,
- DecimalWithPrecisionAndScale<24>, DecimalWithPrecisionAndScale<27>,
- DecimalWithPrecisionAndScale<29>, DecimalWithPrecisionAndScale<32>,
- DecimalWithPrecisionAndScale<34>, DecimalWithPrecisionAndScale<38>>
- TestTypes;
-
-TYPED_TEST_CASE(TestParquetIO, TestTypes);
-
-TYPED_TEST(TestParquetIO, SingleColumnRequiredWrite) {
- std::shared_ptr<Array> values;
- ASSERT_OK(NonNullArray<TypeParam>(SMALL_SIZE, &values));
-
- std::shared_ptr<GroupNode> schema =
- MakeSimpleSchema(*values->type(), Repetition::REQUIRED);
- ASSERT_NO_FATAL_FAILURE(this->WriteColumn(schema, values));
-
- ASSERT_NO_FATAL_FAILURE(this->ReadAndCheckSingleColumnFile(*values));
-}
-
-TYPED_TEST(TestParquetIO, SingleColumnTableRequiredWrite) {
- std::shared_ptr<Array> values;
- ASSERT_OK(NonNullArray<TypeParam>(SMALL_SIZE, &values));
- std::shared_ptr<Table> table = MakeSimpleTable(values, false);
- this->sink_ = std::make_shared<InMemoryOutputStream>();
- ASSERT_OK_NO_THROW(WriteTable(*table, ::arrow::default_memory_pool(), this->sink_,
- values->length(), default_writer_properties()));
-
- std::shared_ptr<Table> out;
- std::unique_ptr<FileReader> reader;
- ASSERT_NO_FATAL_FAILURE(this->ReaderFromSink(&reader));
- ASSERT_NO_FATAL_FAILURE(this->ReadTableFromFile(std::move(reader), &out));
- ASSERT_EQ(1, out->num_columns());
- ASSERT_EQ(100, out->num_rows());
-
- std::shared_ptr<ChunkedArray> chunked_array = out->column(0)->data();
- ASSERT_EQ(1, chunked_array->num_chunks());
-
- internal::AssertArraysEqual(*values, *chunked_array->chunk(0));
-}
-
-TYPED_TEST(TestParquetIO, SingleColumnOptionalReadWrite) {
- // This also tests max_definition_level = 1
- std::shared_ptr<Array> values;
-
- ASSERT_OK(NullableArray<TypeParam>(SMALL_SIZE, 10, kDefaultSeed, &values));
-
- std::shared_ptr<GroupNode> schema =
- MakeSimpleSchema(*values->type(), Repetition::OPTIONAL);
- ASSERT_NO_FATAL_FAILURE(this->WriteColumn(schema, values));
-
- ASSERT_NO_FATAL_FAILURE(this->ReadAndCheckSingleColumnFile(*values));
-}
-
-TYPED_TEST(TestParquetIO, SingleColumnOptionalDictionaryWrite) {
- // Skip tests for BOOL as we don't create dictionaries for it.
- if (TypeParam::type_id == ::arrow::Type::BOOL) {
- return;
- }
-
- std::shared_ptr<Array> values;
-
- ASSERT_OK(NullableArray<TypeParam>(SMALL_SIZE, 10, kDefaultSeed, &values));
-
- Datum out;
- FunctionContext ctx(default_memory_pool());
- ASSERT_OK(DictionaryEncode(&ctx, Datum(values), &out));
- std::shared_ptr<Array> dict_values = MakeArray(out.array());
- std::shared_ptr<GroupNode> schema =
- MakeSimpleSchema(*dict_values->type(), Repetition::OPTIONAL);
- ASSERT_NO_FATAL_FAILURE(this->WriteColumn(schema, dict_values));
-
- ASSERT_NO_FATAL_FAILURE(this->ReadAndCheckSingleColumnFile(*values));
-}
-
-TYPED_TEST(TestParquetIO, SingleColumnRequiredSliceWrite) {
- std::shared_ptr<Array> values;
- ASSERT_OK(NonNullArray<TypeParam>(2 * SMALL_SIZE, &values));
- std::shared_ptr<GroupNode> schema =
- MakeSimpleSchema(*values->type(), Repetition::REQUIRED);
-
- std::shared_ptr<Array> sliced_values = values->Slice(SMALL_SIZE / 2, SMALL_SIZE);
- ASSERT_NO_FATAL_FAILURE(this->WriteColumn(schema, sliced_values));
- ASSERT_NO_FATAL_FAILURE(this->ReadAndCheckSingleColumnFile(*sliced_values));
-
- // Slice offset 1 higher
- sliced_values = values->Slice(SMALL_SIZE / 2 + 1, SMALL_SIZE);
- ASSERT_NO_FATAL_FAILURE(this->WriteColumn(schema, sliced_values));
- ASSERT_NO_FATAL_FAILURE(this->ReadAndCheckSingleColumnFile(*sliced_values));
-}
-
-TYPED_TEST(TestParquetIO, SingleColumnOptionalSliceWrite) {
- std::shared_ptr<Array> values;
- ASSERT_OK(NullableArray<TypeParam>(2 * SMALL_SIZE, SMALL_SIZE, kDefaultSeed, &values));
- std::shared_ptr<GroupNode> schema =
- MakeSimpleSchema(*values->type(), Repetition::OPTIONAL);
-
- std::shared_ptr<Array> sliced_values = values->Slice(SMALL_SIZE / 2, SMALL_SIZE);
- ASSERT_NO_FATAL_FAILURE(this->WriteColumn(schema, sliced_values));
- ASSERT_NO_FATAL_FAILURE(this->ReadAndCheckSingleColumnFile(*sliced_values));
-
- // Slice offset 1 higher, thus different null bitmap.
- sliced_values = values->Slice(SMALL_SIZE / 2 + 1, SMALL_SIZE);
- ASSERT_NO_FATAL_FAILURE(this->WriteColumn(schema, sliced_values));
- ASSERT_NO_FATAL_FAILURE(this->ReadAndCheckSingleColumnFile(*sliced_values));
-}
-
-TYPED_TEST(TestParquetIO, SingleColumnTableOptionalReadWrite) {
- // This also tests max_definition_level = 1
- std::shared_ptr<Array> values;
-
- ASSERT_OK(NullableArray<TypeParam>(SMALL_SIZE, 10, kDefaultSeed, &values));
- std::shared_ptr<Table> table = MakeSimpleTable(values, true);
- ASSERT_NO_FATAL_FAILURE(this->CheckRoundTrip(table));
-}
-
-TYPED_TEST(TestParquetIO, SingleEmptyListsColumnReadWrite) {
- std::shared_ptr<Table> table;
- ASSERT_NO_FATAL_FAILURE(this->PrepareEmptyListsTable(SMALL_SIZE, &table));
- ASSERT_NO_FATAL_FAILURE(this->CheckRoundTrip(table));
-}
-
-TYPED_TEST(TestParquetIO, SingleNullableListNullableColumnReadWrite) {
- std::shared_ptr<Table> table;
- ASSERT_NO_FATAL_FAILURE(this->PrepareListTable(SMALL_SIZE, true, true, 10, &table));
- ASSERT_NO_FATAL_FAILURE(this->CheckRoundTrip(table));
-}
-
-TYPED_TEST(TestParquetIO, SingleRequiredListNullableColumnReadWrite) {
- std::shared_ptr<Table> table;
- ASSERT_NO_FATAL_FAILURE(this->PrepareListTable(SMALL_SIZE, false, true, 10, &table));
- ASSERT_NO_FATAL_FAILURE(this->CheckRoundTrip(table));
-}
-
-TYPED_TEST(TestParquetIO, SingleNullableListRequiredColumnReadWrite) {
- std::shared_ptr<Table> table;
- ASSERT_NO_FATAL_FAILURE(this->PrepareListTable(SMALL_SIZE, true, false, 10, &table));
- ASSERT_NO_FATAL_FAILURE(this->CheckRoundTrip(table));
-}
-
-TYPED_TEST(TestParquetIO, SingleRequiredListRequiredColumnReadWrite) {
- std::shared_ptr<Table> table;
- ASSERT_NO_FATAL_FAILURE(this->PrepareListTable(SMALL_SIZE, false, false, 0, &table));
- ASSERT_NO_FATAL_FAILURE(this->CheckRoundTrip(table));
-}
-
-TYPED_TEST(TestParquetIO, SingleNullableListRequiredListRequiredColumnReadWrite) {
- std::shared_ptr<Table> table;
- ASSERT_NO_FATAL_FAILURE(
- this->PrepareListOfListTable(SMALL_SIZE, true, false, false, 0, &table));
- ASSERT_NO_FATAL_FAILURE(this->CheckRoundTrip(table));
-}
-
-TYPED_TEST(TestParquetIO, SingleColumnRequiredChunkedWrite) {
- std::shared_ptr<Array> values;
- ASSERT_OK(NonNullArray<TypeParam>(SMALL_SIZE, &values));
- int64_t chunk_size = values->length() / 4;
-
- std::shared_ptr<GroupNode> schema =
- MakeSimpleSchema(*values->type(), Repetition::REQUIRED);
- FileWriter writer(default_memory_pool(), this->MakeWriter(schema));
- for (int i = 0; i < 4; i++) {
- ASSERT_OK_NO_THROW(writer.NewRowGroup(chunk_size));
- std::shared_ptr<Array> sliced_array = values->Slice(i * chunk_size, chunk_size);
- ASSERT_OK_NO_THROW(writer.WriteColumnChunk(*sliced_array));
- }
- ASSERT_OK_NO_THROW(writer.Close());
-
- ASSERT_NO_FATAL_FAILURE(this->ReadAndCheckSingleColumnFile(*values));
-}
-
-TYPED_TEST(TestParquetIO, SingleColumnTableRequiredChunkedWrite) {
- std::shared_ptr<Array> values;
- ASSERT_OK(NonNullArray<TypeParam>(LARGE_SIZE, &values));
- std::shared_ptr<Table> table = MakeSimpleTable(values, false);
- this->sink_ = std::make_shared<InMemoryOutputStream>();
- ASSERT_OK_NO_THROW(WriteTable(*table, default_memory_pool(), this->sink_, 512,
- default_writer_properties()));
-
- ASSERT_NO_FATAL_FAILURE(this->ReadAndCheckSingleColumnTable(values));
-}
-
-TYPED_TEST(TestParquetIO, SingleColumnTableRequiredChunkedWriteArrowIO) {
- std::shared_ptr<Array> values;
- ASSERT_OK(NonNullArray<TypeParam>(LARGE_SIZE, &values));
- std::shared_ptr<Table> table = MakeSimpleTable(values, false);
- this->sink_ = std::make_shared<InMemoryOutputStream>();
- auto buffer = AllocateBuffer();
-
- {
- // BufferOutputStream closed on gc
- auto arrow_sink_ = std::make_shared<::arrow::io::BufferOutputStream>(buffer);
- ASSERT_OK_NO_THROW(WriteTable(*table, default_memory_pool(), arrow_sink_, 512,
- default_writer_properties()));
-
- // XXX: Remove this after ARROW-455 completed
- ASSERT_OK(arrow_sink_->Close());
- }
-
- auto pbuffer = std::make_shared<Buffer>(buffer->data(), buffer->size());
-
- auto source = std::make_shared<BufferReader>(pbuffer);
- std::shared_ptr<::arrow::Table> out;
- std::unique_ptr<FileReader> reader;
- ASSERT_OK_NO_THROW(OpenFile(source, ::arrow::default_memory_pool(), &reader));
- ASSERT_NO_FATAL_FAILURE(this->ReadTableFromFile(std::move(reader), &out));
- ASSERT_EQ(1, out->num_columns());
- ASSERT_EQ(values->length(), out->num_rows());
-
- std::shared_ptr<ChunkedArray> chunked_array = out->column(0)->data();
- ASSERT_EQ(1, chunked_array->num_chunks());
-
- internal::AssertArraysEqual(*values, *chunked_array->chunk(0));
-}
-
-TYPED_TEST(TestParquetIO, SingleColumnOptionalChunkedWrite) {
- int64_t chunk_size = SMALL_SIZE / 4;
- std::shared_ptr<Array> values;
-
- ASSERT_OK(NullableArray<TypeParam>(SMALL_SIZE, 10, kDefaultSeed, &values));
-
- std::shared_ptr<GroupNode> schema =
- MakeSimpleSchema(*values->type(), Repetition::OPTIONAL);
- FileWriter writer(::arrow::default_memory_pool(), this->MakeWriter(schema));
- for (int i = 0; i < 4; i++) {
- ASSERT_OK_NO_THROW(writer.NewRowGroup(chunk_size));
- std::shared_ptr<Array> sliced_array = values->Slice(i * chunk_size, chunk_size);
- ASSERT_OK_NO_THROW(writer.WriteColumnChunk(*sliced_array));
- }
- ASSERT_OK_NO_THROW(writer.Close());
-
- ASSERT_NO_FATAL_FAILURE(this->ReadAndCheckSingleColumnFile(*values));
-}
-
-TYPED_TEST(TestParquetIO, SingleColumnTableOptionalChunkedWrite) {
- // This also tests max_definition_level = 1
- std::shared_ptr<Array> values;
-
- ASSERT_OK(NullableArray<TypeParam>(LARGE_SIZE, 100, kDefaultSeed, &values));
- std::shared_ptr<Table> table = MakeSimpleTable(values, true);
- this->sink_ = std::make_shared<InMemoryOutputStream>();
- ASSERT_OK_NO_THROW(WriteTable(*table, ::arrow::default_memory_pool(), this->sink_, 512,
- default_writer_properties()));
-
- ASSERT_NO_FATAL_FAILURE(this->ReadAndCheckSingleColumnTable(values));
-}
-
-TYPED_TEST(TestParquetIO, FileMetaDataWrite) {
- std::shared_ptr<Array> values;
- ASSERT_OK(NonNullArray<TypeParam>(SMALL_SIZE, &values));
- std::shared_ptr<Table> table = MakeSimpleTable(values, false);
- this->sink_ = std::make_shared<InMemoryOutputStream>();
- ASSERT_OK_NO_THROW(WriteTable(*table, ::arrow::default_memory_pool(), this->sink_,
- values->length(), default_writer_properties()));
-
- std::unique_ptr<FileReader> reader;
- ASSERT_NO_FATAL_FAILURE(this->ReaderFromSink(&reader));
- auto metadata = reader->parquet_reader()->metadata();
- ASSERT_EQ(1, metadata->num_columns());
- ASSERT_EQ(100, metadata->num_rows());
-
- this->sink_ = std::make_shared<InMemoryOutputStream>();
-
- ASSERT_OK_NO_THROW(::parquet::arrow::WriteFileMetaData(*metadata, this->sink_.get()));
-
- ASSERT_NO_FATAL_FAILURE(this->ReaderFromSink(&reader));
- auto metadata_written = reader->parquet_reader()->metadata();
- ASSERT_EQ(metadata->size(), metadata_written->size());
- ASSERT_EQ(metadata->num_row_groups(), metadata_written->num_row_groups());
- ASSERT_EQ(metadata->num_rows(), metadata_written->num_rows());
- ASSERT_EQ(metadata->num_columns(), metadata_written->num_columns());
- ASSERT_EQ(metadata->RowGroup(0)->num_rows(), metadata_written->RowGroup(0)->num_rows());
-}
-
-using TestInt96ParquetIO = TestParquetIO<::arrow::TimestampType>;
-
-TEST_F(TestInt96ParquetIO, ReadIntoTimestamp) {
- // This test explicitly tests the conversion from an Impala-style timestamp
- // to a nanoseconds-since-epoch one.
-
- // 2nd January 1970, 11:35min 145738543ns
- Int96 day;
- day.value[2] = UINT32_C(2440589);
- int64_t seconds = (11 * 60 + 35) * 60;
- *(reinterpret_cast<int64_t*>(&(day.value))) =
- seconds * INT64_C(1000) * INT64_C(1000) * INT64_C(1000) + 145738543;
- // Compute the corresponding nanosecond timestamp
- struct tm datetime;
- memset(&datetime, 0, sizeof(struct tm));
- datetime.tm_year = 70;
- datetime.tm_mon = 0;
- datetime.tm_mday = 2;
- datetime.tm_hour = 11;
- datetime.tm_min = 35;
- struct tm epoch;
- memset(&epoch, 0, sizeof(struct tm));
-
- epoch.tm_year = 70;
- epoch.tm_mday = 1;
- // Nanoseconds since the epoch
- int64_t val = lrint(difftime(mktime(&datetime), mktime(&epoch))) * INT64_C(1000000000);
- val += 145738543;
-
- std::vector<std::shared_ptr<schema::Node>> fields(
- {schema::PrimitiveNode::Make("int96", Repetition::REQUIRED, ParquetType::INT96)});
- std::shared_ptr<schema::GroupNode> schema = std::static_pointer_cast<GroupNode>(
- schema::GroupNode::Make("schema", Repetition::REQUIRED, fields));
-
- // We cannot write this column with Arrow, so we have to use the plain parquet-cpp API
- // to write an Int96 file.
- this->sink_ = std::make_shared<InMemoryOutputStream>();
- auto writer = ParquetFileWriter::Open(this->sink_, schema);
- RowGroupWriter* rg_writer = writer->AppendRowGroup();
- ColumnWriter* c_writer = rg_writer->NextColumn();
- auto typed_writer = dynamic_cast<TypedColumnWriter<Int96Type>*>(c_writer);
- ASSERT_NE(typed_writer, nullptr);
- typed_writer->WriteBatch(1, nullptr, nullptr, &day);
- c_writer->Close();
- rg_writer->Close();
- writer->Close();
-
- ::arrow::TimestampBuilder builder(::arrow::timestamp(TimeUnit::NANO),
- ::arrow::default_memory_pool());
- ASSERT_OK(builder.Append(val));
- std::shared_ptr<Array> values;
- ASSERT_OK(builder.Finish(&values));
- ASSERT_NO_FATAL_FAILURE(this->ReadAndCheckSingleColumnFile(*values));
-}
-
-using TestUInt32ParquetIO = TestParquetIO<::arrow::UInt32Type>;
-
-TEST_F(TestUInt32ParquetIO, Parquet_2_0_Compability) {
- // This also tests max_definition_level = 1
- std::shared_ptr<Array> values;
-
- ASSERT_OK(NullableArray<::arrow::UInt32Type>(LARGE_SIZE, 100, kDefaultSeed, &values));
- std::shared_ptr<Table> table = MakeSimpleTable(values, true);
-
- // Parquet 2.0 roundtrip should yield an uint32_t column again
- this->sink_ = std::make_shared<InMemoryOutputStream>();
- std::shared_ptr<::parquet::WriterProperties> properties =
- ::parquet::WriterProperties::Builder()
- .version(ParquetVersion::PARQUET_2_0)
- ->build();
- ASSERT_OK_NO_THROW(
- WriteTable(*table, default_memory_pool(), this->sink_, 512, properties));
- ASSERT_NO_FATAL_FAILURE(this->ReadAndCheckSingleColumnTable(values));
-}
-
-TEST_F(TestUInt32ParquetIO, Parquet_1_0_Compability) {
- // This also tests max_definition_level = 1
- std::shared_ptr<Array> arr;
- ASSERT_OK(NullableArray<::arrow::UInt32Type>(LARGE_SIZE, 100, kDefaultSeed, &arr));
-
- std::shared_ptr<::arrow::UInt32Array> values =
- std::dynamic_pointer_cast<::arrow::UInt32Array>(arr);
-
- std::shared_ptr<Table> table = MakeSimpleTable(values, true);
-
- // Parquet 1.0 returns an int64_t column as there is no way to tell a Parquet 1.0
- // reader that a column is unsigned.
- this->sink_ = std::make_shared<InMemoryOutputStream>();
- std::shared_ptr<::parquet::WriterProperties> properties =
- ::parquet::WriterProperties::Builder()
- .version(ParquetVersion::PARQUET_1_0)
- ->build();
- ASSERT_OK_NO_THROW(
- WriteTable(*table, ::arrow::default_memory_pool(), this->sink_, 512, properties));
-
- std::shared_ptr<ResizableBuffer> int64_data = AllocateBuffer();
- {
- ASSERT_OK(int64_data->Resize(sizeof(int64_t) * values->length()));
- auto int64_data_ptr = reinterpret_cast<int64_t*>(int64_data->mutable_data());
- auto uint32_data_ptr = reinterpret_cast<const uint32_t*>(values->values()->data());
- const auto cast_uint32_to_int64 = [](uint32_t value) {
- return static_cast<int64_t>(value);
- };
- std::transform(uint32_data_ptr, uint32_data_ptr + values->length(), int64_data_ptr,
- cast_uint32_to_int64);
- }
-
- std::vector<std::shared_ptr<Buffer>> buffers{values->null_bitmap(), int64_data};
- auto arr_data = std::make_shared<::arrow::ArrayData>(::arrow::int64(), values->length(),
- buffers, values->null_count());
- std::shared_ptr<Array> expected_values = MakeArray(arr_data);
- ASSERT_NE(expected_values, NULLPTR);
-
- const auto& expected = static_cast<const ::arrow::Int64Array&>(*expected_values);
- ASSERT_GT(values->length(), 0);
- ASSERT_EQ(values->length(), expected.length());
-
- // TODO(phillipc): Is there a better way to compare these two arrays?
- // AssertArraysEqual requires the same type, but we only care about values in this case
- for (int i = 0; i < expected.length(); ++i) {
- const bool value_is_valid = values->IsValid(i);
- const bool expected_value_is_valid = expected.IsValid(i);
-
- ASSERT_EQ(expected_value_is_valid, value_is_valid);
-
- if (value_is_valid) {
- uint32_t value = values->Value(i);
- int64_t expected_value = expected.Value(i);
- ASSERT_EQ(expected_value, static_cast<int64_t>(value));
- }
- }
-}
-
-using TestStringParquetIO = TestParquetIO<::arrow::StringType>;
-
-TEST_F(TestStringParquetIO, EmptyStringColumnRequiredWrite) {
- std::shared_ptr<Array> values;
- ::arrow::StringBuilder builder;
- for (size_t i = 0; i < SMALL_SIZE; i++) {
- ASSERT_OK(builder.Append(""));
- }
- ASSERT_OK(builder.Finish(&values));
- std::shared_ptr<Table> table = MakeSimpleTable(values, false);
- this->sink_ = std::make_shared<InMemoryOutputStream>();
- ASSERT_OK_NO_THROW(WriteTable(*table, ::arrow::default_memory_pool(), this->sink_,
- values->length(), default_writer_properties()));
-
- std::shared_ptr<Table> out;
- std::unique_ptr<FileReader> reader;
- ASSERT_NO_FATAL_FAILURE(this->ReaderFromSink(&reader));
- ASSERT_NO_FATAL_FAILURE(this->ReadTableFromFile(std::move(reader), &out));
- ASSERT_EQ(1, out->num_columns());
- ASSERT_EQ(100, out->num_rows());
-
- std::shared_ptr<ChunkedArray> chunked_array = out->column(0)->data();
- ASSERT_EQ(1, chunked_array->num_chunks());
-
- internal::AssertArraysEqual(*values, *chunked_array->chunk(0));
-}
-
-using TestNullParquetIO = TestParquetIO<::arrow::NullType>;
-
-TEST_F(TestNullParquetIO, NullColumn) {
- for (int32_t num_rows : {0, SMALL_SIZE}) {
- std::shared_ptr<Array> values = std::make_shared<::arrow::NullArray>(num_rows);
- std::shared_ptr<Table> table = MakeSimpleTable(values, true /* nullable */);
- this->sink_ = std::make_shared<InMemoryOutputStream>();
-
- const int64_t chunk_size = std::max(static_cast<int64_t>(1), table->num_rows());
- ASSERT_OK_NO_THROW(WriteTable(*table, ::arrow::default_memory_pool(), this->sink_,
- chunk_size, default_writer_properties()));
-
- std::shared_ptr<Table> out;
- std::unique_ptr<FileReader> reader;
- ASSERT_NO_FATAL_FAILURE(this->ReaderFromSink(&reader));
- ASSERT_NO_FATAL_FAILURE(this->ReadTableFromFile(std::move(reader), &out));
- ASSERT_EQ(1, out->num_columns());
- ASSERT_EQ(num_rows, out->num_rows());
-
- std::shared_ptr<ChunkedArray> chunked_array = out->column(0)->data();
- ASSERT_EQ(1, chunked_array->num_chunks());
- internal::AssertArraysEqual(*values, *chunked_array->chunk(0));
- }
-}
-
-TEST_F(TestNullParquetIO, NullListColumn) {
- std::vector<int32_t> offsets1 = {0};
- std::vector<int32_t> offsets2 = {0, 2, 2, 3, 115};
- for (std::vector<int32_t> offsets : {offsets1, offsets2}) {
- std::shared_ptr<Array> offsets_array, values_array, list_array;
- ::arrow::ArrayFromVector<::arrow::Int32Type, int32_t>(offsets, &offsets_array);
- values_array = std::make_shared<::arrow::NullArray>(offsets.back());
- ASSERT_OK(::arrow::ListArray::FromArrays(*offsets_array, *values_array,
- default_memory_pool(), &list_array));
-
- std::shared_ptr<Table> table = MakeSimpleTable(list_array, false /* nullable */);
- this->sink_ = std::make_shared<InMemoryOutputStream>();
-
- const int64_t chunk_size = std::max(static_cast<int64_t>(1), table->num_rows());
- ASSERT_OK_NO_THROW(WriteTable(*table, ::arrow::default_memory_pool(), this->sink_,
- chunk_size, default_writer_properties()));
-
- std::shared_ptr<Table> out;
- std::unique_ptr<FileReader> reader;
- this->ReaderFromSink(&reader);
- this->ReadTableFromFile(std::move(reader), &out);
- ASSERT_EQ(1, out->num_columns());
- ASSERT_EQ(offsets.size() - 1, out->num_rows());
-
- std::shared_ptr<ChunkedArray> chunked_array = out->column(0)->data();
- ASSERT_EQ(1, chunked_array->num_chunks());
- internal::AssertArraysEqual(*list_array, *chunked_array->chunk(0));
- }
-}
-
-TEST_F(TestNullParquetIO, NullDictionaryColumn) {
- std::shared_ptr<Array> values = std::make_shared<::arrow::NullArray>(0);
- std::shared_ptr<Array> indices =
- std::make_shared<::arrow::Int8Array>(SMALL_SIZE, nullptr, nullptr, SMALL_SIZE);
- std::shared_ptr<::arrow::DictionaryType> dict_type =
- std::make_shared<::arrow::DictionaryType>(::arrow::int8(), values);
- std::shared_ptr<Array> dict_values =
- std::make_shared<::arrow::DictionaryArray>(dict_type, indices);
- std::shared_ptr<Table> table = MakeSimpleTable(dict_values, true);
- this->sink_ = std::make_shared<InMemoryOutputStream>();
- ASSERT_OK_NO_THROW(WriteTable(*table, ::arrow::default_memory_pool(), this->sink_,
- dict_values->length(), default_writer_properties()));
-
- std::shared_ptr<Table> out;
- std::unique_ptr<FileReader> reader;
- ASSERT_NO_FATAL_FAILURE(this->ReaderFromSink(&reader));
- ASSERT_NO_FATAL_FAILURE(this->ReadTableFromFile(std::move(reader), &out));
- ASSERT_EQ(1, out->num_columns());
- ASSERT_EQ(100, out->num_rows());
-
- std::shared_ptr<ChunkedArray> chunked_array = out->column(0)->data();
- ASSERT_EQ(1, chunked_array->num_chunks());
-
- std::shared_ptr<Array> expected_values =
- std::make_shared<::arrow::NullArray>(SMALL_SIZE);
- internal::AssertArraysEqual(*expected_values, *chunked_array->chunk(0));
-}
-
-template <typename T>
-using ParquetCDataType = typename ParquetDataType<T>::c_type;
-
-template <typename T>
-struct c_type_trait {
- using ArrowCType = typename T::c_type;
-};
-
-template <>
-struct c_type_trait<::arrow::BooleanType> {
- using ArrowCType = uint8_t;
-};
-
-template <typename TestType>
-class TestPrimitiveParquetIO : public TestParquetIO<TestType> {
- public:
- typedef typename c_type_trait<TestType>::ArrowCType T;
-
- void MakeTestFile(std::vector<T>& values, int num_chunks,
- std::unique_ptr<FileReader>* reader) {
- TestType dummy;
-
- std::shared_ptr<GroupNode> schema = MakeSimpleSchema(dummy, Repetition::REQUIRED);
- std::unique_ptr<ParquetFileWriter> file_writer = this->MakeWriter(schema);
- size_t chunk_size = values.size() / num_chunks;
- // Convert to Parquet's expected physical type
- std::vector<uint8_t> values_buffer(sizeof(ParquetCDataType<TestType>) *
- values.size());
- auto values_parquet =
- reinterpret_cast<ParquetCDataType<TestType>*>(values_buffer.data());
- std::copy(values.cbegin(), values.cend(), values_parquet);
- for (int i = 0; i < num_chunks; i++) {
- auto row_group_writer = file_writer->AppendRowGroup();
- auto column_writer =
- static_cast<ParquetWriter<TestType>*>(row_group_writer->NextColumn());
- ParquetCDataType<TestType>* data = values_parquet + i * chunk_size;
- column_writer->WriteBatch(chunk_size, nullptr, nullptr, data);
- column_writer->Close();
- row_group_writer->Close();
- }
- file_writer->Close();
- this->ReaderFromSink(reader);
- }
-
- void CheckSingleColumnRequiredTableRead(int num_chunks) {
- std::vector<T> values(SMALL_SIZE, test_traits<TestType>::value);
- std::unique_ptr<FileReader> file_reader;
- ASSERT_NO_FATAL_FAILURE(MakeTestFile(values, num_chunks, &file_reader));
-
- std::shared_ptr<Table> out;
- this->ReadTableFromFile(std::move(file_reader), &out);
- ASSERT_EQ(1, out->num_columns());
- ASSERT_EQ(SMALL_SIZE, out->num_rows());
-
- std::shared_ptr<ChunkedArray> chunked_array = out->column(0)->data();
- ASSERT_EQ(1, chunked_array->num_chunks());
- ExpectArrayT<TestType>(values.data(), chunked_array->chunk(0).get());
- }
-
- void CheckSingleColumnRequiredRead(int num_chunks) {
- std::vector<T> values(SMALL_SIZE, test_traits<TestType>::value);
- std::unique_ptr<FileReader> file_reader;
- ASSERT_NO_FATAL_FAILURE(MakeTestFile(values, num_chunks, &file_reader));
-
- std::shared_ptr<Array> out;
- this->ReadSingleColumnFile(std::move(file_reader), &out);
-
- ExpectArrayT<TestType>(values.data(), out.get());
- }
-};
-
-typedef ::testing::Types<::arrow::BooleanType, ::arrow::UInt8Type, ::arrow::Int8Type,
- ::arrow::UInt16Type, ::arrow::Int16Type, ::arrow::UInt32Type,
- ::arrow::Int32Type, ::arrow::UInt64Type, ::arrow::Int64Type,
- ::arrow::FloatType, ::arrow::DoubleType>
- PrimitiveTestTypes;
-
-TYPED_TEST_CASE(TestPrimitiveParquetIO, PrimitiveTestTypes);
-
-TYPED_TEST(TestPrimitiveParquetIO, SingleColumnRequiredRead) {
- ASSERT_NO_FATAL_FAILURE(this->CheckSingleColumnRequiredRead(1));
-}
-
-TYPED_TEST(TestPrimitiveParquetIO, SingleColumnRequiredTableRead) {
- ASSERT_NO_FATAL_FAILURE(this->CheckSingleColumnRequiredTableRead(1));
-}
-
-TYPED_TEST(TestPrimitiveParquetIO, SingleColumnRequiredChunkedRead) {
- ASSERT_NO_FATAL_FAILURE(this->CheckSingleColumnRequiredRead(4));
-}
-
-TYPED_TEST(TestPrimitiveParquetIO, SingleColumnRequiredChunkedTableRead) {
- ASSERT_NO_FATAL_FAILURE(this->CheckSingleColumnRequiredTableRead(4));
-}
-
-void MakeDateTimeTypesTable(std::shared_ptr<Table>* out, bool nanos_as_micros = false) {
- using ::arrow::ArrayFromVector;
-
- std::vector<bool> is_valid = {true, true, true, false, true, true};
-
- // These are only types that roundtrip without modification
- auto f0 = field("f0", ::arrow::date32());
- auto f1 = field("f1", ::arrow::timestamp(TimeUnit::MILLI));
- auto f2 = field("f2", ::arrow::timestamp(TimeUnit::MICRO));
- std::shared_ptr<::arrow::Field> f3;
- if (nanos_as_micros) {
- f3 = field("f3", ::arrow::timestamp(TimeUnit::MICRO));
- } else {
- f3 = field("f3", ::arrow::timestamp(TimeUnit::NANO));
- }
- auto f4 = field("f4", ::arrow::time32(TimeUnit::MILLI));
- auto f5 = field("f5", ::arrow::time64(TimeUnit::MICRO));
- std::shared_ptr<::arrow::Schema> schema(new ::arrow::Schema({f0, f1, f2, f3, f4, f5}));
-
- std::vector<int32_t> t32_values = {1489269000, 1489270000, 1489271000,
- 1489272000, 1489272000, 1489273000};
- std::vector<int64_t> t64_values = {1489269000000, 1489270000000, 1489271000000,
- 1489272000000, 1489272000000, 1489273000000};
- std::vector<int64_t> t64_us_values = {1489269000, 1489270000, 1489271000,
- 1489272000, 1489272000, 1489273000};
-
- std::shared_ptr<Array> a0, a1, a2, a3, a4, a5;
- ArrayFromVector<::arrow::Date32Type, int32_t>(f0->type(), is_valid, t32_values, &a0);
- ArrayFromVector<::arrow::TimestampType, int64_t>(f1->type(), is_valid, t64_values, &a1);
- ArrayFromVector<::arrow::TimestampType, int64_t>(f2->type(), is_valid, t64_values, &a2);
- if (nanos_as_micros) {
- ArrayFromVector<::arrow::TimestampType, int64_t>(f3->type(), is_valid, t64_us_values,
- &a3);
- } else {
- ArrayFromVector<::arrow::TimestampType, int64_t>(f3->type(), is_valid, t64_values,
- &a3);
- }
- ArrayFromVector<::arrow::Time32Type, int32_t>(f4->type(), is_valid, t32_values, &a4);
- ArrayFromVector<::arrow::Time64Type, int64_t>(f5->type(), is_valid, t64_values, &a5);
-
- std::vector<std::shared_ptr<::arrow::Column>> columns = {
- std::make_shared<Column>("f0", a0), std::make_shared<Column>("f1", a1),
- std::make_shared<Column>("f2", a2), std::make_shared<Column>("f3", a3),
- std::make_shared<Column>("f4", a4), std::make_shared<Column>("f5", a5)};
- *out = Table::Make(schema, columns);
-}
-
-TEST(TestArrowReadWrite, DateTimeTypes) {
- std::shared_ptr<Table> table;
- MakeDateTimeTypesTable(&table);
-
- // Use deprecated INT96 type
- std::shared_ptr<Table> result;
- ASSERT_NO_FATAL_FAILURE(DoSimpleRoundtrip(
- table, false /* use_threads */, table->num_rows(), {}, &result,
- ArrowWriterProperties::Builder().enable_deprecated_int96_timestamps()->build()));
-
- ASSERT_NO_FATAL_FAILURE(::arrow::AssertTablesEqual(*table, *result));
-
- // Cast nanaoseconds to microseconds and use INT64 physical type
- ASSERT_NO_FATAL_FAILURE(
- DoSimpleRoundtrip(table, false /* use_threads */, table->num_rows(), {}, &result));
- std::shared_ptr<Table> expected;
- MakeDateTimeTypesTable(&table, true);
-
- ASSERT_NO_FATAL_FAILURE(::arrow::AssertTablesEqual(*table, *result));
-}
-
-TEST(TestArrowReadWrite, CoerceTimestamps) {
- using ::arrow::ArrayFromVector;
- using ::arrow::field;
-
- // PARQUET-1078, coerce Arrow timestamps to either TIMESTAMP_MILLIS or TIMESTAMP_MICROS
- std::vector<bool> is_valid = {true, true, true, false, true, true};
-
- auto t_s = ::arrow::timestamp(TimeUnit::SECOND);
- auto t_ms = ::arrow::timestamp(TimeUnit::MILLI);
- auto t_us = ::arrow::timestamp(TimeUnit::MICRO);
- auto t_ns = ::arrow::timestamp(TimeUnit::NANO);
-
- std::vector<int64_t> s_values = {1489269, 1489270, 1489271, 1489272, 1489272, 1489273};
- std::vector<int64_t> ms_values = {1489269000, 1489270000, 1489271000,
- 1489272001, 1489272000, 1489273000};
- std::vector<int64_t> us_values = {1489269000000, 1489270000000, 1489271000000,
- 1489272000001, 1489272000000, 1489273000000};
- std::vector<int64_t> ns_values = {1489269000000000LL, 1489270000000000LL,
- 1489271000000000LL, 1489272000000001LL,
- 1489272000000000LL, 1489273000000000LL};
-
- std::shared_ptr<Array> a_s, a_ms, a_us, a_ns;
- ArrayFromVector<::arrow::TimestampType, int64_t>(t_s, is_valid, s_values, &a_s);
- ArrayFromVector<::arrow::TimestampType, int64_t>(t_ms, is_valid, ms_values, &a_ms);
- ArrayFromVector<::arrow::TimestampType, int64_t>(t_us, is_valid, us_values, &a_us);
- ArrayFromVector<::arrow::TimestampType, int64_t>(t_ns, is_valid, ns_values, &a_ns);
-
- // Input table, all data as is
- auto s1 = std::shared_ptr<::arrow::Schema>(
- new ::arrow::Schema({field("f_s", t_s), field("f_ms", t_ms), field("f_us", t_us),
- field("f_ns", t_ns)}));
- auto input = Table::Make(
- s1,
- {std::make_shared<Column>("f_s", a_s), std::make_shared<Column>("f_ms", a_ms),
- std::make_shared<Column>("f_us", a_us), std::make_shared<Column>("f_ns", a_ns)});
-
- // Result when coercing to milliseconds
- auto s2 = std::shared_ptr<::arrow::Schema>(
- new ::arrow::Schema({field("f_s", t_ms), field("f_ms", t_ms), field("f_us", t_ms),
- field("f_ns", t_ms)}));
- auto ex_milli_result = Table::Make(
- s2,
- {std::make_shared<Column>("f_s", a_ms), std::make_shared<Column>("f_ms", a_ms),
- std::make_shared<Column>("f_us", a_ms), std::make_shared<Column>("f_ns", a_ms)});
-
- // Result when coercing to microseconds
- auto s3 = std::shared_ptr<::arrow::Schema>(
- new ::arrow::Schema({field("f_s", t_us), field("f_ms", t_us), field("f_us", t_us),
- field("f_ns", t_us)}));
- auto ex_micro_result = Table::Make(
- s3,
- {std::make_shared<Column>("f_s", a_us), std::make_shared<Column>("f_ms", a_us),
- std::make_shared<Column>("f_us", a_us), std::make_shared<Column>("f_ns", a_us)});
-
- std::shared_ptr<Table> milli_result;
- ASSERT_NO_FATAL_FAILURE(DoSimpleRoundtrip(
- input, false /* use_threads */, input->num_rows(), {}, &milli_result,
- ArrowWriterProperties::Builder().coerce_timestamps(TimeUnit::MILLI)->build()));
-
- ASSERT_NO_FATAL_FAILURE(::arrow::AssertTablesEqual(*ex_milli_result, *milli_result));
-
- std::shared_ptr<Table> micro_result;
- ASSERT_NO_FATAL_FAILURE(DoSimpleRoundtrip(
- input, false /* use_threads */, input->num_rows(), {}, µ_result,
- ArrowWriterProperties::Builder().coerce_timestamps(TimeUnit::MICRO)->build()));
- ASSERT_NO_FATAL_FAILURE(::arrow::AssertTablesEqual(*ex_micro_result, *micro_result));
-}
-
-TEST(TestArrowReadWrite, CoerceTimestampsLosePrecision) {
- using ::arrow::ArrayFromVector;
- using ::arrow::field;
-
- // PARQUET-1078, coerce Arrow timestamps to either TIMESTAMP_MILLIS or TIMESTAMP_MICROS
- std::vector<bool> is_valid = {true, true, true, false, true, true};
-
- auto t_s = ::arrow::timestamp(TimeUnit::SECOND);
- auto t_ms = ::arrow::timestamp(TimeUnit::MILLI);
- auto t_us = ::arrow::timestamp(TimeUnit::MICRO);
- auto t_ns = ::arrow::timestamp(TimeUnit::NANO);
-
- std::vector<int64_t> s_values = {1489269, 1489270, 1489271, 1489272, 1489272, 1489273};
- std::vector<int64_t> ms_values = {1489269001, 1489270001, 1489271001,
- 1489272001, 1489272001, 1489273001};
- std::vector<int64_t> us_values = {1489269000001, 1489270000001, 1489271000001,
- 1489272000001, 1489272000001, 1489273000001};
- std::vector<int64_t> ns_values = {1489269000000001LL, 1489270000000001LL,
- 1489271000000001LL, 1489272000000001LL,
- 1489272000000001LL, 1489273000000001LL};
-
- std::shared_ptr<Array> a_s, a_ms, a_us, a_ns;
- ArrayFromVector<::arrow::TimestampType, int64_t>(t_s, is_valid, s_values, &a_s);
- ArrayFromVector<::arrow::TimestampType, int64_t>(t_ms, is_valid, ms_values, &a_ms);
- ArrayFromVector<::arrow::TimestampType, int64_t>(t_us, is_valid, us_values, &a_us);
- ArrayFromVector<::arrow::TimestampType, int64_t>(t_ns, is_valid, ns_values, &a_ns);
-
- auto s1 = std::shared_ptr<::arrow::Schema>(new ::arrow::Schema({field("f_s", t_s)}));
- auto s2 = std::shared_ptr<::arrow::Schema>(new ::arrow::Schema({field("f_ms", t_ms)}));
- auto s3 = std::shared_ptr<::arrow::Schema>(new ::arrow::Schema({field("f_us", t_us)}));
- auto s4 = std::shared_ptr<::arrow::Schema>(new ::arrow::Schema({field("f_ns", t_ns)}));
-
- auto c1 = std::make_shared<Column>("f_s", a_s);
- auto c2 = std::make_shared<Column>("f_ms", a_ms);
- auto c3 = std::make_shared<Column>("f_us", a_us);
- auto c4 = std::make_shared<Column>("f_ns", a_ns);
-
- auto t1 = Table::Make(s1, {c1});
- auto t2 = Table::Make(s2, {c2});
- auto t3 = Table::Make(s3, {c3});
- auto t4 = Table::Make(s4, {c4});
-
- auto sink = std::make_shared<InMemoryOutputStream>();
-
- // OK to write to millis
- auto coerce_millis =
- (ArrowWriterProperties::Builder().coerce_timestamps(TimeUnit::MILLI)->build());
- ASSERT_OK_NO_THROW(WriteTable(*t1, ::arrow::default_memory_pool(), sink, 10,
- default_writer_properties(), coerce_millis));
- ASSERT_OK_NO_THROW(WriteTable(*t2, ::arrow::default_memory_pool(), sink, 10,
- default_writer_properties(), coerce_millis));
-
- // Loss of precision
- ASSERT_RAISES(Invalid, WriteTable(*t3, ::arrow::default_memory_pool(), sink, 10,
- default_writer_properties(), coerce_millis));
- ASSERT_RAISES(Invalid, WriteTable(*t4, ::arrow::default_memory_pool(), sink, 10,
- default_writer_properties(), coerce_millis));
-
- // OK to write micros to micros
- auto coerce_micros =
- (ArrowWriterProperties::Builder().coerce_timestamps(TimeUnit::MICRO)->build());
- ASSERT_OK_NO_THROW(WriteTable(*t3, ::arrow::default_memory_pool(), sink, 10,
- default_writer_properties(), coerce_micros));
-
- // Loss of precision
- ASSERT_RAISES(Invalid, WriteTable(*t4, ::arrow::default_memory_pool(), sink, 10,
- default_writer_properties(), coerce_micros));
-}
-
-TEST(TestArrowReadWrite, ConvertedDateTimeTypes) {
- using ::arrow::ArrayFromVector;
-
- std::vector<bool> is_valid = {true, true, true, false, true, true};
-
- auto f0 = field("f0", ::arrow::date64());
- auto f1 = field("f1", ::arrow::time32(TimeUnit::SECOND));
- auto f2 = field("f2", ::arrow::date64());
- auto f3 = field("f3", ::arrow::time32(TimeUnit::SECOND));
-
- auto schema = ::arrow::schema({f0, f1, f2, f3});
-
- std::vector<int64_t> a0_values = {1489190400000, 1489276800000, 1489363200000,
- 1489449600000, 1489536000000, 1489622400000};
- std::vector<int32_t> a1_values = {0, 1, 2, 3, 4, 5};
-
- std::shared_ptr<Array> a0, a1, a0_nonnull, a1_nonnull, x0, x1, x0_nonnull, x1_nonnull;
-
- ArrayFromVector<::arrow::Date64Type, int64_t>(f0->type(), is_valid, a0_values, &a0);
- ArrayFromVector<::arrow::Date64Type, int64_t>(f0->type(), a0_values, &a0_nonnull);
-
- ArrayFromVector<::arrow::Time32Type, int32_t>(f1->type(), is_valid, a1_values, &a1);
- ArrayFromVector<::arrow::Time32Type, int32_t>(f1->type(), a1_values, &a1_nonnull);
-
- std::vector<std::shared_ptr<::arrow::Column>> columns = {
- std::make_shared<Column>("f0", a0), std::make_shared<Column>("f1", a1),
- std::make_shared<Column>("f2", a0_nonnull),
- std::make_shared<Column>("f3", a1_nonnull)};
- auto table = Table::Make(schema, columns);
-
- // Expected schema and values
- auto e0 = field("f0", ::arrow::date32());
- auto e1 = field("f1", ::arrow::time32(TimeUnit::MILLI));
- auto e2 = field("f2", ::arrow::date32());
- auto e3 = field("f3", ::arrow::time32(TimeUnit::MILLI));
- auto ex_schema = ::arrow::schema({e0, e1, e2, e3});
-
- std::vector<int32_t> x0_values = {17236, 17237, 17238, 17239, 17240, 17241};
- std::vector<int32_t> x1_values = {0, 1000, 2000, 3000, 4000, 5000};
- ArrayFromVector<::arrow::Date32Type, int32_t>(e0->type(), is_valid, x0_values, &x0);
- ArrayFromVector<::arrow::Date32Type, int32_t>(e0->type(), x0_values, &x0_nonnull);
-
- ArrayFromVector<::arrow::Time32Type, int32_t>(e1->type(), is_valid, x1_values, &x1);
- ArrayFromVector<::arrow::Time32Type, int32_t>(e1->type(), x1_values, &x1_nonnull);
-
- std::vector<std::shared_ptr<::arrow::Column>> ex_columns = {
- std::make_shared<Column>("f0", x0), std::make_shared<Column>("f1", x1),
- std::make_shared<Column>("f2", x0_nonnull),
- std::make_shared<Column>("f3", x1_nonnull)};
- auto ex_table = Table::Make(ex_schema, ex_columns);
-
- std::shared_ptr<Table> result;
- ASSERT_NO_FATAL_FAILURE(
- DoSimpleRoundtrip(table, false /* use_threads */, table->num_rows(), {}, &result));
-
- ASSERT_NO_FATAL_FAILURE(::arrow::AssertTablesEqual(*ex_table, *result));
-}
-
-// Regression for ARROW-2802
-TEST(TestArrowReadWrite, CoerceTimestampsAndSupportDeprecatedInt96) {
- using ::arrow::Column;
- using ::arrow::default_memory_pool;
- using ::arrow::Field;
- using ::arrow::Schema;
- using ::arrow::Table;
- using ::arrow::TimestampBuilder;
- using ::arrow::TimestampType;
- using ::arrow::TimeUnit;
-
- auto timestamp_type = std::make_shared<TimestampType>(TimeUnit::NANO);
-
- TimestampBuilder builder(timestamp_type, default_memory_pool());
- for (std::int64_t ii = 0; ii < 10; ++ii) {
- ASSERT_OK(builder.Append(1000000000L * ii));
- }
- std::shared_ptr<Array> values;
- ASSERT_OK(builder.Finish(&values));
-
- std::vector<std::shared_ptr<Field>> fields;
- auto field = std::make_shared<Field>("nanos", timestamp_type);
- fields.emplace_back(field);
-
- auto schema = std::make_shared<Schema>(fields);
-
- std::vector<std::shared_ptr<Column>> columns;
- auto column = std::make_shared<Column>("nanos", values);
- columns.emplace_back(column);
-
- auto table = Table::Make(schema, columns);
-
- auto arrow_writer_properties = ArrowWriterProperties::Builder()
- .coerce_timestamps(TimeUnit::MICRO)
- ->enable_deprecated_int96_timestamps()
- ->build();
-
- std::shared_ptr<Table> result;
- DoSimpleRoundtrip(table, false /* use_threads */, table->num_rows(), {}, &result,
- arrow_writer_properties);
-
- ASSERT_EQ(table->num_columns(), result->num_columns());
- ASSERT_EQ(table->num_rows(), result->num_rows());
-
- auto actual_column = result->column(0);
- auto data = actual_column->data();
- auto expected_values =
- static_cast<::arrow::NumericArray<TimestampType>*>(values.get())->raw_values();
- for (int ii = 0; ii < data->num_chunks(); ++ii) {
- auto chunk =
- static_cast<::arrow::NumericArray<TimestampType>*>(data->chunk(ii).get());
- auto values = chunk->raw_values();
- for (int64_t jj = 0; jj < chunk->length(); ++jj, ++expected_values) {
- // Check that the nanos have been converted to micros
- ASSERT_EQ(*expected_values / 1000, values[jj]);
- }
- }
-}
-
-void MakeDoubleTable(int num_columns, int num_rows, int nchunks,
- std::shared_ptr<Table>* out) {
- std::shared_ptr<::arrow::Column> column;
- std::vector<std::shared_ptr<::arrow::Column>> columns(num_columns);
- std::vector<std::shared_ptr<::arrow::Field>> fields(num_columns);
-
- for (int i = 0; i < num_columns; ++i) {
- std::vector<std::shared_ptr<Array>> arrays;
- std::shared_ptr<Array> values;
- ASSERT_OK(NullableArray<::arrow::DoubleType>(num_rows, num_rows / 10,
- static_cast<uint32_t>(i), &values));
- std::stringstream ss;
- ss << "col" << i;
-
- for (int j = 0; j < nchunks; ++j) {
- arrays.push_back(values);
- }
- column = MakeColumn(ss.str(), arrays, true);
-
- columns[i] = column;
- fields[i] = column->field();
- }
- auto schema = std::make_shared<::arrow::Schema>(fields);
- *out = Table::Make(schema, columns);
-}
-
-void MakeListArray(int num_rows, int max_value_length,
- std::shared_ptr<::DataType>* out_type,
- std::shared_ptr<Array>* out_array) {
- std::vector<int32_t> length_draws;
- randint(num_rows, 0, max_value_length, &length_draws);
-
- std::vector<int32_t> offset_values;
-
- // Make sure some of them are length 0
- int32_t total_elements = 0;
- for (size_t i = 0; i < length_draws.size(); ++i) {
- if (length_draws[i] < max_value_length / 10) {
- length_draws[i] = 0;
- }
- offset_values.push_back(total_elements);
- total_elements += length_draws[i];
- }
- offset_values.push_back(total_elements);
-
- std::vector<int8_t> value_draws;
- randint(total_elements, 0, 100, &value_draws);
-
- std::vector<bool> is_valid;
- random_is_valid(total_elements, 0.1, &is_valid);
-
- std::shared_ptr<Array> values, offsets;
- ::arrow::ArrayFromVector<::arrow::Int8Type, int8_t>(::arrow::int8(), is_valid,
- value_draws, &values);
- ::arrow::ArrayFromVector<::arrow::Int32Type, int32_t>(offset_values, &offsets);
-
- ASSERT_OK(::arrow::ListArray::FromArrays(*offsets, *values, default_memory_pool(),
- out_array));
-
- *out_type = ::arrow::list(::arrow::int8());
-}
-
-TEST(TestArrowReadWrite, MultithreadedRead) {
- const int num_columns = 20;
- const int num_rows = 1000;
- const bool use_threads = true;
-
- std::shared_ptr<Table> table;
- ASSERT_NO_FATAL_FAILURE(MakeDoubleTable(num_columns, num_rows, 1, &table));
-
- std::shared_ptr<Table> result;
- ASSERT_NO_FATAL_FAILURE(
- DoSimpleRoundtrip(table, use_threads, table->num_rows(), {}, &result));
-
- ASSERT_NO_FATAL_FAILURE(::arrow::AssertTablesEqual(*table, *result));
-}
-
-TEST(TestArrowReadWrite, ReadSingleRowGroup) {
- const int num_columns = 20;
- const int num_rows = 1000;
-
- std::shared_ptr<Table> table;
- ASSERT_NO_FATAL_FAILURE(MakeDoubleTable(num_columns, num_rows, 1, &table));
-
- std::shared_ptr<Buffer> buffer;
- ASSERT_NO_FATAL_FAILURE(WriteTableToBuffer(table, num_rows / 2,
- default_arrow_writer_properties(), &buffer));
-
- std::unique_ptr<FileReader> reader;
- ASSERT_OK_NO_THROW(OpenFile(std::make_shared<BufferReader>(buffer),
- ::arrow::default_memory_pool(),
- ::parquet::default_reader_properties(), nullptr, &reader));
-
- ASSERT_EQ(2, reader->num_row_groups());
-
- std::shared_ptr<Table> r1, r2, r3, r4;
- // Read everything
- ASSERT_OK_NO_THROW(reader->ReadRowGroup(0, &r1));
- ASSERT_OK_NO_THROW(reader->RowGroup(1)->ReadTable(&r2));
- ASSERT_OK_NO_THROW(reader->ReadRowGroups({0, 1}, &r3));
- ASSERT_OK_NO_THROW(reader->ReadRowGroups({1}, &r4));
-
- std::shared_ptr<Table> concatenated;
-
- ASSERT_OK(ConcatenateTables({r1, r2}, &concatenated));
- ASSERT_TRUE(table->Equals(*concatenated));
-
- ASSERT_TRUE(table->Equals(*r3));
- ASSERT_TRUE(r2->Equals(*r4));
- ASSERT_OK(ConcatenateTables({r1, r4}, &concatenated));
- ASSERT_TRUE(table->Equals(*concatenated));
-}
-
-TEST(TestArrowReadWrite, GetRecordBatchReader) {
- const int num_columns = 20;
- const int num_rows = 1000;
-
- std::shared_ptr<Table> table;
- ASSERT_NO_FATAL_FAILURE(MakeDoubleTable(num_columns, num_rows, 1, &table));
-
- std::shared_ptr<Buffer> buffer;
- ASSERT_NO_FATAL_FAILURE(WriteTableToBuffer(table, num_rows / 2,
- default_arrow_writer_properties(), &buffer));
-
- std::unique_ptr<FileReader> reader;
- ASSERT_OK_NO_THROW(OpenFile(std::make_shared<BufferReader>(buffer),
- ::arrow::default_memory_pool(),
- ::parquet::default_reader_properties(), nullptr, &reader));
-
- std::shared_ptr<::arrow::RecordBatchReader> rb_reader;
- ASSERT_OK_NO_THROW(reader->GetRecordBatchReader({0, 1}, &rb_reader));
-
- std::shared_ptr<::arrow::RecordBatch> batch;
-
- ASSERT_OK(rb_reader->ReadNext(&batch));
- ASSERT_EQ(500, batch->num_rows());
- ASSERT_EQ(20, batch->num_columns());
-
- ASSERT_OK(rb_reader->ReadNext(&batch));
- ASSERT_EQ(500, batch->num_rows());
- ASSERT_EQ(20, batch->num_columns());
-
- ASSERT_OK(rb_reader->ReadNext(&batch));
- ASSERT_EQ(nullptr, batch);
-}
-
-TEST(TestArrowReadWrite, ScanContents) {
- const int num_columns = 20;
- const int num_rows = 1000;
-
- std::shared_ptr<Table> table;
- ASSERT_NO_FATAL_FAILURE(MakeDoubleTable(num_columns, num_rows, 1, &table));
-
- std::shared_ptr<Buffer> buffer;
- ASSERT_NO_FATAL_FAILURE(WriteTableToBuffer(table, num_rows / 2,
- default_arrow_writer_properties(), &buffer));
-
- std::unique_ptr<FileReader> reader;
- ASSERT_OK_NO_THROW(OpenFile(std::make_shared<BufferReader>(buffer),
- ::arrow::default_memory_pool(),
- ::parquet::default_reader_properties(), nullptr, &reader));
-
- int64_t num_rows_returned = 0;
- ASSERT_OK_NO_THROW(reader->ScanContents({}, 256, &num_rows_returned));
- ASSERT_EQ(num_rows, num_rows_returned);
-
- ASSERT_OK_NO_THROW(reader->ScanContents({0, 1, 2}, 256, &num_rows_returned));
- ASSERT_EQ(num_rows, num_rows_returned);
-}
-
-TEST(TestArrowReadWrite, ReadColumnSubset) {
- const int num_columns = 20;
- const int num_rows = 1000;
- const bool use_threads = true;
-
- std::shared_ptr<Table> table;
- ASSERT_NO_FATAL_FAILURE(MakeDoubleTable(num_columns, num_rows, 1, &table));
-
- std::shared_ptr<Table> result;
- std::vector<int> column_subset = {0, 4, 8, 10};
- ASSERT_NO_FATAL_FAILURE(
- DoSimpleRoundtrip(table, use_threads, table->num_rows(), column_subset, &result));
-
- std::vector<std::shared_ptr<::arrow::Column>> ex_columns;
- std::vector<std::shared_ptr<::arrow::Field>> ex_fields;
- for (int i : column_subset) {
- ex_columns.push_back(table->column(i));
- ex_fields.push_back(table->column(i)->field());
- }
-
- auto ex_schema = ::arrow::schema(ex_fields);
- auto expected = Table::Make(ex_schema, ex_columns);
- ASSERT_NO_FATAL_FAILURE(::arrow::AssertTablesEqual(*expected, *result));
-}
-
-TEST(TestArrowReadWrite, ListLargeRecords) {
- // PARQUET-1308: This test passed on Linux when num_rows was smaller
- const int num_rows = 2000;
-
- std::shared_ptr<Array> list_array;
- std::shared_ptr<::DataType> list_type;
-
- MakeListArray(num_rows, 20, &list_type, &list_array);
-
- auto schema = ::arrow::schema({::arrow::field("a", list_type)});
-
- std::shared_ptr<Table> table = Table::Make(schema, {list_array});
-
- std::shared_ptr<Buffer> buffer;
- ASSERT_NO_FATAL_FAILURE(
- WriteTableToBuffer(table, 100, default_arrow_writer_properties(), &buffer));
-
- std::unique_ptr<FileReader> reader;
- ASSERT_OK_NO_THROW(OpenFile(std::make_shared<BufferReader>(buffer),
- ::arrow::default_memory_pool(),
- ::parquet::default_reader_properties(), nullptr, &reader));
-
- // Read everything
- std::shared_ptr<Table> result;
- ASSERT_OK_NO_THROW(reader->ReadTable(&result));
- ASSERT_NO_FATAL_FAILURE(::arrow::AssertTablesEqual(*table, *result));
-
- // Read chunked
- ASSERT_OK_NO_THROW(OpenFile(std::make_shared<BufferReader>(buffer),
- ::arrow::default_memory_pool(),
- ::parquet::default_reader_properties(), nullptr, &reader));
-
- std::unique_ptr<ColumnReader> col_reader;
- ASSERT_OK(reader->GetColumn(0, &col_reader));
-
- std::vector<std::shared_ptr<Array>> pieces;
- for (int i = 0; i < num_rows; ++i) {
- std::shared_ptr<Array> piece;
- ASSERT_OK(col_reader->NextBatch(1, &piece));
- ASSERT_EQ(1, piece->length());
- pieces.push_back(piece);
- }
- auto chunked = std::make_shared<::arrow::ChunkedArray>(pieces);
-
- auto chunked_col =
- std::make_shared<::arrow::Column>(table->schema()->field(0), chunked);
- std::vector<std::shared_ptr<::arrow::Column>> columns = {chunked_col};
- auto chunked_table = Table::Make(table->schema(), columns);
-
- ASSERT_TRUE(table->Equals(*chunked_table));
-}
-
-typedef std::function<void(int, std::shared_ptr<::DataType>*, std::shared_ptr<Array>*)>
- ArrayFactory;
-
-template <typename ArrowType>
-struct GenerateArrayFunctor {
- explicit GenerateArrayFunctor(double pct_null = 0.1) : pct_null(pct_null) {}
-
- void operator()(int length, std::shared_ptr<::DataType>* type,
- std::shared_ptr<Array>* array) {
- using T = typename ArrowType::c_type;
-
- // TODO(wesm): generate things other than integers
- std::vector<T> draws;
- randint(length, 0, 100, &draws);
-
- std::vector<bool> is_valid;
- random_is_valid(length, this->pct_null, &is_valid);
-
- *type = ::arrow::TypeTraits<ArrowType>::type_singleton();
- ::arrow::ArrayFromVector<ArrowType, T>(*type, is_valid, draws, array);
- }
-
- double pct_null;
-};
-
-typedef std::function<void(int, std::shared_ptr<::DataType>*, std::shared_ptr<Array>*)>
- ArrayFactory;
-
-auto GenerateInt32 = [](int length, std::shared_ptr<::DataType>* type,
- std::shared_ptr<Array>* array) {
- GenerateArrayFunctor<::arrow::Int32Type> func;
- func(length, type, array);
-};
-
-auto GenerateList = [](int length, std::shared_ptr<::DataType>* type,
- std::shared_ptr<Array>* array) {
- MakeListArray(length, 100, type, array);
-};
-
-TEST(TestArrowReadWrite, TableWithChunkedColumns) {
- std::vector<ArrayFactory> functions = {GenerateInt32, GenerateList};
-
- std::vector<int> chunk_sizes = {2, 4, 10, 2};
- const int64_t total_length = 18;
-
- for (const auto& datagen_func : functions) {
- ::arrow::ArrayVector arrays;
- std::shared_ptr<Array> arr;
- std::shared_ptr<::DataType> type;
- datagen_func(total_length, &type, &arr);
-
- int64_t offset = 0;
- for (int chunk_size : chunk_sizes) {
- arrays.push_back(arr->Slice(offset, chunk_size));
- offset += chunk_size;
- }
-
- auto field = ::arrow::field("fname", type);
- auto schema = ::arrow::schema({field});
- auto col = std::make_shared<::arrow::Column>(field, arrays);
- auto table = Table::Make(schema, {col});
-
- ASSERT_NO_FATAL_FAILURE(CheckSimpleRoundtrip(table, 2));
- ASSERT_NO_FATAL_FAILURE(CheckSimpleRoundtrip(table, 3));
- ASSERT_NO_FATAL_FAILURE(CheckSimpleRoundtrip(table, 10));
- }
-}
-
-TEST(TestArrowReadWrite, TableWithDuplicateColumns) {
- // See ARROW-1974
- using ::arrow::ArrayFromVector;
-
- auto f0 = field("duplicate", ::arrow::int8());
- auto f1 = field("duplicate", ::arrow::int16());
- auto schema = ::arrow::schema({f0, f1});
-
- std::vector<int8_t> a0_values = {1, 2, 3};
- std::vector<int16_t> a1_values = {14, 15, 16};
-
- std::shared_ptr<Array> a0, a1;
-
- ArrayFromVector<::arrow::Int8Type, int8_t>(a0_values, &a0);
- ArrayFromVector<::arrow::Int16Type, int16_t>(a1_values, &a1);
-
- auto table = Table::Make(schema, {std::make_shared<Column>(f0->name(), a0),
- std::make_shared<Column>(f1->name(), a1)});
- ASSERT_NO_FATAL_FAILURE(CheckSimpleRoundtrip(table, table->num_rows()));
-}
-
-TEST(TestArrowReadWrite, DictionaryColumnChunkedWrite) {
- // This is a regression test for this:
- //
- // https://issues.apache.org/jira/browse/ARROW-1938
- //
- // As of the writing of this test, columns of type
- // dictionary are written as their raw/expanded values.
- // The regression was that the whole column was being
- // written for each chunk.
- using ::arrow::ArrayFromVector;
-
- std::vector<std::string> values = {"first", "second", "third"};
- auto type = ::arrow::utf8();
- std::shared_ptr<Array> dict_values;
- ArrayFromVector<::arrow::StringType, std::string>(values, &dict_values);
-
- auto dict_type = ::arrow::dictionary(::arrow::int32(), dict_values);
- auto f0 = field("dictionary", dict_type);
- std::vector<std::shared_ptr<::arrow::Field>> fields;
- fields.emplace_back(f0);
- auto schema = ::arrow::schema(fields);
-
- std::shared_ptr<Array> f0_values, f1_values;
- ArrayFromVector<::arrow::Int32Type, int32_t>({0, 1, 0, 2, 1}, &f0_values);
- ArrayFromVector<::arrow::Int32Type, int32_t>({2, 0, 1, 0, 2}, &f1_values);
- ::arrow::ArrayVector dict_arrays = {
- std::make_shared<::arrow::DictionaryArray>(dict_type, f0_values),
- std::make_shared<::arrow::DictionaryArray>(dict_type, f1_values)};
-
- std::vector<std::shared_ptr<::arrow::Column>> columns;
- auto column = MakeColumn("column", dict_arrays, false);
- columns.emplace_back(column);
-
- auto table = Table::Make(schema, columns);
-
- std::shared_ptr<Table> result;
- DoSimpleRoundtrip(table, 1,
- // Just need to make sure that we make
- // a chunk size that is smaller than the
- // total number of values
- 2, {}, &result);
-
- std::vector<std::string> expected_values = {"first", "second", "first", "third",
- "second", "third", "first", "second",
- "first", "third"};
- columns.clear();
-
- std::shared_ptr<Array> expected_array;
- ArrayFromVector<::arrow::StringType, std::string>(expected_values, &expected_array);
-
- // The column name gets changed on output to the name of the
- // field, and it also turns into a nullable column
- columns.emplace_back(MakeColumn("dictionary", expected_array, true));
-
- fields.clear();
- fields.emplace_back(::arrow::field("dictionary", ::arrow::utf8()));
- schema = ::arrow::schema(fields);
-
- auto expected_table = Table::Make(schema, columns);
-
- ::arrow::AssertTablesEqual(*expected_table, *result, false);
-}
-
-TEST(TestArrowWrite, CheckChunkSize) {
- const int num_columns = 2;
- const int num_rows = 128;
- const int64_t chunk_size = 0; // note the chunk_size is 0
- std::shared_ptr<Table> table;
- ASSERT_NO_FATAL_FAILURE(MakeDoubleTable(num_columns, num_rows, 1, &table));
-
- auto sink = std::make_shared<InMemoryOutputStream>();
-
- ASSERT_RAISES(Invalid,
- WriteTable(*table, ::arrow::default_memory_pool(), sink, chunk_size));
-}
-
-class TestNestedSchemaRead : public ::testing::TestWithParam<Repetition::type> {
- protected:
- // make it *3 to make it easily divisible by 3
- const int NUM_SIMPLE_TEST_ROWS = SMALL_SIZE * 3;
- std::shared_ptr<::arrow::Int32Array> values_array_ = nullptr;
-
- void InitReader() {
- std::shared_ptr<Buffer> buffer = nested_parquet_->GetBuffer();
- ASSERT_OK_NO_THROW(
- OpenFile(std::make_shared<BufferReader>(buffer), ::arrow::default_memory_pool(),
- ::parquet::default_reader_properties(), nullptr, &reader_));
- }
-
- void InitNewParquetFile(const std::shared_ptr<GroupNode>& schema, int num_rows) {
- nested_parquet_ = std::make_shared<InMemoryOutputStream>();
-
- writer_ = parquet::ParquetFileWriter::Open(nested_parquet_, schema,
- default_writer_properties());
- row_group_writer_ = writer_->AppendRowGroup();
- }
-
- void FinalizeParquetFile() {
- row_group_writer_->Close();
- writer_->Close();
- }
-
- void MakeValues(int num_rows) {
- std::shared_ptr<Array> arr;
- ASSERT_OK(NullableArray<::arrow::Int32Type>(num_rows, 0, kDefaultSeed, &arr));
- values_array_ = std::dynamic_pointer_cast<::arrow::Int32Array>(arr);
- }
-
- void WriteColumnData(size_t num_rows, int16_t* def_levels, int16_t* rep_levels,
- int32_t* values) {
- auto typed_writer =
- static_cast<TypedColumnWriter<Int32Type>*>(row_group_writer_->NextColumn());
- typed_writer->WriteBatch(num_rows, def_levels, rep_levels, values);
- }
-
- void ValidateArray(const Array& array, size_t expected_nulls) {
- ASSERT_EQ(array.length(), values_array_->length());
- ASSERT_EQ(array.null_count(), expected_nulls);
- // Also independently count the nulls
- auto local_null_count = 0;
- for (int i = 0; i < array.length(); i++) {
- if (array.IsNull(i)) {
- local_null_count++;
- }
- }
- ASSERT_EQ(local_null_count, expected_nulls);
- }
-
- void ValidateColumnArray(const ::arrow::Int32Array& array, size_t expected_nulls) {
- ValidateArray(array, expected_nulls);
- int j = 0;
- for (int i = 0; i < values_array_->length(); i++) {
- if (array.IsNull(i)) {
- continue;
- }
- ASSERT_EQ(array.Value(i), values_array_->Value(j));
- j++;
- }
- }
-
- void ValidateTableArrayTypes(const Table& table) {
- for (int i = 0; i < table.num_columns(); i++) {
- const std::shared_ptr<::arrow::Field> schema_field = table.schema()->field(i);
- const std::shared_ptr<Column> column = table.column(i);
- // Compare with the column field
- ASSERT_TRUE(schema_field->Equals(column->field()));
- // Compare with the array type
- ASSERT_TRUE(schema_field->type()->Equals(column->data()->chunk(0)->type()));
- }
- }
-
- // A parquet with a simple nested schema
- void CreateSimpleNestedParquet(Repetition::type struct_repetition) {
- std::vector<NodePtr> parquet_fields;
- // TODO(itaiin): We are using parquet low-level file api to create the nested parquet
- // this needs to change when a nested writes are implemented
-
- // create the schema:
- // <struct_repetition> group group1 {
- // required int32 leaf1;
- // optional int32 leaf2;
- // }
- // required int32 leaf3;
-
- parquet_fields.push_back(GroupNode::Make(
- "group1", struct_repetition,
- {PrimitiveNode::Make("leaf1", Repetition::REQUIRED, ParquetType::INT32),
- PrimitiveNode::Make("leaf2", Repetition::OPTIONAL, ParquetType::INT32)}));
- parquet_fields.push_back(
- PrimitiveNode::Make("leaf3", Repetition::REQUIRED, ParquetType::INT32));
-
- auto schema_node = GroupNode::Make("schema", Repetition::REQUIRED, parquet_fields);
-
- // Create definition levels for the different columns that contain interleaved
- // nulls and values at all nesting levels
-
- // definition levels for optional fields
- std::vector<int16_t> leaf1_def_levels(NUM_SIMPLE_TEST_ROWS);
- std::vector<int16_t> leaf2_def_levels(NUM_SIMPLE_TEST_ROWS);
- std::vector<int16_t> leaf3_def_levels(NUM_SIMPLE_TEST_ROWS);
- for (int i = 0; i < NUM_SIMPLE_TEST_ROWS; i++) {
- // leaf1 is required within the optional group1, so it is only null
- // when the group is null
- leaf1_def_levels[i] = (i % 3 == 0) ? 0 : 1;
- // leaf2 is optional, can be null in the primitive (def-level 1) or
- // struct level (def-level 0)
- leaf2_def_levels[i] = static_cast<int16_t>(i % 3);
- // leaf3 is required
- leaf3_def_levels[i] = 0;
- }
-
- std::vector<int16_t> rep_levels(NUM_SIMPLE_TEST_ROWS, 0);
-
- // Produce values for the columns
- MakeValues(NUM_SIMPLE_TEST_ROWS);
- int32_t* values = reinterpret_cast<int32_t*>(values_array_->values()->mutable_data());
-
- // Create the actual parquet file
- InitNewParquetFile(std::static_pointer_cast<GroupNode>(schema_node),
- NUM_SIMPLE_TEST_ROWS);
-
- // leaf1 column
- WriteColumnData(NUM_SIMPLE_TEST_ROWS, leaf1_def_levels.data(), rep_levels.data(),
- values);
- // leaf2 column
- WriteColumnData(NUM_SIMPLE_TEST_ROWS, leaf2_def_levels.data(), rep_levels.data(),
- values);
- // leaf3 column
- WriteColumnData(NUM_SIMPLE_TEST_ROWS, leaf3_def_levels.data(), rep_levels.data(),
- values);
-
- FinalizeParquetFile();
- InitReader();
- }
-
- NodePtr CreateSingleTypedNestedGroup(int index, int depth, int num_children,
- Repetition::type node_repetition,
- ParquetType::type leaf_type) {
- std::vector<NodePtr> children;
-
- for (int i = 0; i < num_children; i++) {
- if (depth <= 1) {
- children.push_back(PrimitiveNode::Make("leaf", node_repetition, leaf_type));
- } else {
- children.push_back(CreateSingleTypedNestedGroup(i, depth - 1, num_children,
- node_repetition, leaf_type));
- }
- }
-
- std::stringstream ss;
- ss << "group-" << depth << "-" << index;
- return NodePtr(GroupNode::Make(ss.str(), node_repetition, children));
- }
-
- // A deeply nested schema
- void CreateMultiLevelNestedParquet(int num_trees, int tree_depth, int num_children,
- int num_rows, Repetition::type node_repetition) {
- // Create the schema
- std::vector<NodePtr> parquet_fields;
- for (int i = 0; i < num_trees; i++) {
- parquet_fields.push_back(CreateSingleTypedNestedGroup(
- i, tree_depth, num_children, node_repetition, ParquetType::INT32));
- }
- auto schema_node = GroupNode::Make("schema", Repetition::REQUIRED, parquet_fields);
-
- int num_columns = num_trees * static_cast<int>((std::pow(num_children, tree_depth)));
-
- std::vector<int16_t> def_levels;
- std::vector<int16_t> rep_levels;
-
- int num_levels = 0;
- while (num_levels < num_rows) {
- if (node_repetition == Repetition::REQUIRED) {
- def_levels.push_back(0); // all are required
- } else {
- int16_t level = static_cast<int16_t>(num_levels % (tree_depth + 2));
- def_levels.push_back(level); // all are optional
- }
- rep_levels.push_back(0); // none is repeated
- ++num_levels;
- }
-
- // Produce values for the columns
- MakeValues(num_rows);
- int32_t* values = reinterpret_cast<int32_t*>(values_array_->values()->mutable_data());
-
- // Create the actual parquet file
- InitNewParquetFile(std::static_pointer_cast<GroupNode>(schema_node), num_rows);
-
- for (int i = 0; i < num_columns; i++) {
- WriteColumnData(num_rows, def_levels.data(), rep_levels.data(), values);
- }
- FinalizeParquetFile();
- InitReader();
- }
-
- class DeepParquetTestVisitor : public ArrayVisitor {
- public:
- DeepParquetTestVisitor(Repetition::type node_repetition,
- std::shared_ptr<::arrow::Int32Array> expected)
- : node_repetition_(node_repetition), expected_(expected) {}
-
- Status Validate(std::shared_ptr<Array> tree) { return tree->Accept(this); }
-
- virtual Status Visit(const ::arrow::Int32Array& array) {
- if (node_repetition_ == Repetition::REQUIRED) {
- if (!array.Equals(expected_)) {
- return Status::Invalid("leaf array data mismatch");
- }
- } else if (node_repetition_ == Repetition::OPTIONAL) {
- if (array.length() != expected_->length()) {
- return Status::Invalid("Bad leaf array length");
- }
- // expect only 1 value every `depth` row
- if (array.null_count() != SMALL_SIZE) {
- return Status::Invalid("Unexpected null count");
- }
- } else {
- return Status::NotImplemented("Unsupported repetition");
- }
- return Status::OK();
- }
-
- virtual Status Visit(const ::arrow::StructArray& array) {
- for (int32_t i = 0; i < array.num_fields(); ++i) {
- auto child = array.field(i);
- if (node_repetition_ == Repetition::REQUIRED) {
- RETURN_NOT_OK(child->Accept(this));
- } else if (node_repetition_ == Repetition::OPTIONAL) {
- // Null count Must be a multiple of SMALL_SIZE
- if (array.null_count() % SMALL_SIZE != 0) {
- return Status::Invalid("Unexpected struct null count");
- }
- } else {
- return Status::NotImplemented("Unsupported repetition");
- }
- }
- return Status::OK();
- }
-
- private:
- Repetition::type node_repetition_;
- std::shared_ptr<::arrow::Int32Array> expected_;
- };
-
- std::shared_ptr<InMemoryOutputStream> nested_parquet_;
- std::unique_ptr<FileReader> reader_;
- std::unique_ptr<ParquetFileWriter> writer_;
- RowGroupWriter* row_group_writer_;
-};
-
-TEST_F(TestNestedSchemaRead, ReadIntoTableFull) {
- ASSERT_NO_FATAL_FAILURE(CreateSimpleNestedParquet(Repetition::OPTIONAL));
-
- std::shared_ptr<Table> table;
- ASSERT_OK_NO_THROW(reader_->ReadTable(&table));
- ASSERT_EQ(table->num_rows(), NUM_SIMPLE_TEST_ROWS);
- ASSERT_EQ(table->num_columns(), 2);
- ASSERT_EQ(table->schema()->field(0)->type()->num_children(), 2);
- ASSERT_NO_FATAL_FAILURE(ValidateTableArrayTypes(*table));
-
- auto struct_field_array =
- std::static_pointer_cast<::arrow::StructArray>(table->column(0)->data()->chunk(0));
- auto leaf1_array =
- std::static_pointer_cast<::arrow::Int32Array>(struct_field_array->field(0));
- auto leaf2_array =
- std::static_pointer_cast<::arrow::Int32Array>(struct_field_array->field(1));
- auto leaf3_array =
- std::static_pointer_cast<::arrow::Int32Array>(table->column(1)->data()->chunk(0));
-
- // validate struct and leaf arrays
-
- // validate struct array
- ASSERT_NO_FATAL_FAILURE(ValidateArray(*struct_field_array, NUM_SIMPLE_TEST_ROWS / 3));
- // validate leaf1
- ASSERT_NO_FATAL_FAILURE(ValidateColumnArray(*leaf1_array, NUM_SIMPLE_TEST_ROWS / 3));
- // validate leaf2
- ASSERT_NO_FATAL_FAILURE(
- ValidateColumnArray(*leaf2_array, NUM_SIMPLE_TEST_ROWS * 2 / 3));
- // validate leaf3
- ASSERT_NO_FATAL_FAILURE(ValidateColumnArray(*leaf3_array, 0));
-}
-
-TEST_F(TestNestedSchemaRead, ReadTablePartial) {
- ASSERT_NO_FATAL_FAILURE(CreateSimpleNestedParquet(Repetition::OPTIONAL));
- std::shared_ptr<Table> table;
-
- // columns: {group1.leaf1, leaf3}
- ASSERT_OK_NO_THROW(reader_->ReadTable({0, 2}, &table));
- ASSERT_EQ(table->num_rows(), NUM_SIMPLE_TEST_ROWS);
- ASSERT_EQ(table->num_columns(), 2);
- ASSERT_EQ(table->schema()->field(0)->name(), "group1");
- ASSERT_EQ(table->schema()->field(1)->name(), "leaf3");
- ASSERT_EQ(table->schema()->field(0)->type()->num_children(), 1);
- ASSERT_NO_FATAL_FAILURE(ValidateTableArrayTypes(*table));
-
- // columns: {group1.leaf1, group1.leaf2}
- ASSERT_OK_NO_THROW(reader_->ReadTable({0, 1}, &table));
- ASSERT_EQ(table->num_rows(), NUM_SIMPLE_TEST_ROWS);
- ASSERT_EQ(table->num_columns(), 1);
- ASSERT_EQ(table->schema()->field(0)->name(), "group1");
- ASSERT_EQ(table->schema()->field(0)->type()->num_children(), 2);
- ASSERT_NO_FATAL_FAILURE(ValidateTableArrayTypes(*table));
-
- // columns: {leaf3}
- ASSERT_OK_NO_THROW(reader_->ReadTable({2}, &table));
- ASSERT_EQ(table->num_rows(), NUM_SIMPLE_TEST_ROWS);
- ASSERT_EQ(table->num_columns(), 1);
- ASSERT_EQ(table->schema()->field(0)->name(), "leaf3");
- ASSERT_EQ(table->schema()->field(0)->type()->num_children(), 0);
- ASSERT_NO_FATAL_FAILURE(ValidateTableArrayTypes(*table));
-
- // Test with different ordering
- ASSERT_OK_NO_THROW(reader_->ReadTable({2, 0}, &table));
- ASSERT_EQ(table->num_rows(), NUM_SIMPLE_TEST_ROWS);
- ASSERT_EQ(table->num_columns(), 2);
- ASSERT_EQ(table->schema()->field(0)->name(), "leaf3");
- ASSERT_EQ(table->schema()->field(1)->name(), "group1");
- ASSERT_EQ(table->schema()->field(1)->type()->num_children(), 1);
- ASSERT_NO_FATAL_FAILURE(ValidateTableArrayTypes(*table));
-}
-
-TEST_F(TestNestedSchemaRead, StructAndListTogetherUnsupported) {
- ASSERT_NO_FATAL_FAILURE(CreateSimpleNestedParquet(Repetition::REPEATED));
- std::shared_ptr<Table> table;
- ASSERT_RAISES(NotImplemented, reader_->ReadTable(&table));
-}
-
-TEST_P(TestNestedSchemaRead, DeepNestedSchemaRead) {
-#ifdef PARQUET_VALGRIND
- const int num_trees = 3;
- const int depth = 3;
-#else
- const int num_trees = 10;
- const int depth = 5;
-#endif
- const int num_children = 3;
- int num_rows = SMALL_SIZE * (depth + 2);
- ASSERT_NO_FATAL_FAILURE(CreateMultiLevelNestedParquet(num_trees, depth, num_children,
- num_rows, GetParam()));
- std::shared_ptr<Table> table;
- ASSERT_OK_NO_THROW(reader_->ReadTable(&table));
- ASSERT_EQ(table->num_columns(), num_trees);
- ASSERT_EQ(table->num_rows(), num_rows);
-
- DeepParquetTestVisitor visitor(GetParam(), values_array_);
- for (int i = 0; i < table->num_columns(); i++) {
- auto tree = table->column(i)->data()->chunk(0);
- ASSERT_OK_NO_THROW(visitor.Validate(tree));
- }
-}
-
-INSTANTIATE_TEST_CASE_P(Repetition_type, TestNestedSchemaRead,
- ::testing::Values(Repetition::REQUIRED, Repetition::OPTIONAL));
-
-TEST(TestImpalaConversion, NanosecondToImpala) {
- // June 20, 2017 16:32:56 and 123456789 nanoseconds
- int64_t nanoseconds = INT64_C(1497976376123456789);
- Int96 expected = {{UINT32_C(632093973), UINT32_C(13871), UINT32_C(2457925)}};
- Int96 calculated;
- internal::NanosecondsToImpalaTimestamp(nanoseconds, &calculated);
- ASSERT_EQ(expected, calculated);
-}
-
-TEST(TestArrowReaderAdHoc, Int96BadMemoryAccess) {
- // PARQUET-995
- std::string dir_string(test::get_data_dir());
- std::stringstream ss;
- ss << dir_string << "/"
- << "alltypes_plain.parquet";
- auto path = ss.str();
-
- auto pool = ::arrow::default_memory_pool();
-
- std::unique_ptr<FileReader> arrow_reader;
- ASSERT_NO_THROW(
- arrow_reader.reset(new FileReader(pool, ParquetFileReader::OpenFile(path, false))));
- std::shared_ptr<::arrow::Table> table;
- ASSERT_OK_NO_THROW(arrow_reader->ReadTable(&table));
-}
-
-class TestArrowReaderAdHocSpark
- : public ::testing::TestWithParam<
- std::tuple<std::string, std::shared_ptr<::DataType>>> {};
-
-TEST_P(TestArrowReaderAdHocSpark, ReadDecimals) {
- std::string path(test::get_data_dir());
-
- std::string filename;
- std::shared_ptr<::DataType> decimal_type;
- std::tie(filename, decimal_type) = GetParam();
-
- path += "/" + filename;
- ASSERT_GT(path.size(), 0);
-
- auto pool = ::arrow::default_memory_pool();
-
- std::unique_ptr<FileReader> arrow_reader;
- ASSERT_NO_THROW(
- arrow_reader.reset(new FileReader(pool, ParquetFileReader::OpenFile(path, false))));
- std::shared_ptr<::arrow::Table> table;
- ASSERT_OK_NO_THROW(arrow_reader->ReadTable(&table));
-
- ASSERT_EQ(1, table->num_columns());
-
- constexpr int32_t expected_length = 24;
-
- auto value_column = table->column(0);
- ASSERT_EQ(expected_length, value_column->length());
-
- auto raw_array = value_column->data();
- ASSERT_EQ(1, raw_array->num_chunks());
-
- auto chunk = raw_array->chunk(0);
-
- std::shared_ptr<Array> expected_array;
-
- ::arrow::Decimal128Builder builder(decimal_type, pool);
-
- for (int32_t i = 0; i < expected_length; ++i) {
- ::arrow::Decimal128 value((i + 1) * 100);
- ASSERT_OK(builder.Append(value));
- }
- ASSERT_OK(builder.Finish(&expected_array));
-
- internal::AssertArraysEqual(*expected_array, *chunk);
-}
-
-INSTANTIATE_TEST_CASE_P(
- ReadDecimals, TestArrowReaderAdHocSpark,
- ::testing::Values(
- std::make_tuple("int32_decimal.parquet", ::arrow::decimal(4, 2)),
- std::make_tuple("int64_decimal.parquet", ::arrow::decimal(10, 2)),
- std::make_tuple("fixed_length_decimal.parquet", ::arrow::decimal(25, 2)),
- std::make_tuple("fixed_length_decimal_legacy.parquet", ::arrow::decimal(13, 2))));
-
-} // namespace arrow
-
-} // namespace parquet
diff --git a/src/parquet/arrow/arrow-schema-test.cc b/src/parquet/arrow/arrow-schema-test.cc
deleted file mode 100644
index cb2b850..0000000
--- a/src/parquet/arrow/arrow-schema-test.cc
+++ /dev/null
@@ -1,852 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#include <memory>
-#include <vector>
-
-#include "gtest/gtest.h"
-
-#include "parquet/arrow/schema.h"
-
-#include "arrow/api.h"
-#include "arrow/test-util.h"
-
-using arrow::ArrayFromVector;
-using arrow::Field;
-using arrow::TimeUnit;
-
-using ParquetType = parquet::Type;
-using parquet::LogicalType;
-using parquet::Repetition;
-using parquet::schema::GroupNode;
-using parquet::schema::NodePtr;
-using parquet::schema::PrimitiveNode;
-
-namespace parquet {
-
-namespace arrow {
-
-const auto BOOL = ::arrow::boolean();
-const auto UINT8 = ::arrow::uint8();
-const auto INT32 = ::arrow::int32();
-const auto INT64 = ::arrow::int64();
-const auto FLOAT = ::arrow::float32();
-const auto DOUBLE = ::arrow::float64();
-const auto UTF8 = ::arrow::utf8();
-const auto TIMESTAMP_MS = ::arrow::timestamp(TimeUnit::MILLI);
-const auto TIMESTAMP_US = ::arrow::timestamp(TimeUnit::MICRO);
-const auto TIMESTAMP_NS = ::arrow::timestamp(TimeUnit::NANO);
-const auto BINARY = ::arrow::binary();
-const auto DECIMAL_8_4 = std::make_shared<::arrow::Decimal128Type>(8, 4);
-
-class TestConvertParquetSchema : public ::testing::Test {
- public:
- virtual void SetUp() {}
-
- void CheckFlatSchema(const std::shared_ptr<::arrow::Schema>& expected_schema) {
- ASSERT_EQ(expected_schema->num_fields(), result_schema_->num_fields());
- for (int i = 0; i < expected_schema->num_fields(); ++i) {
- auto lhs = result_schema_->field(i);
- auto rhs = expected_schema->field(i);
- EXPECT_TRUE(lhs->Equals(rhs))
- << i << " " << lhs->ToString() << " != " << rhs->ToString();
- }
- }
-
- ::arrow::Status ConvertSchema(const std::vector<NodePtr>& nodes) {
- NodePtr schema = GroupNode::Make("schema", Repetition::REPEATED, nodes);
- descr_.Init(schema);
- return FromParquetSchema(&descr_, &result_schema_);
- }
-
- ::arrow::Status ConvertSchema(const std::vector<NodePtr>& nodes,
- const std::vector<int>& column_indices) {
- NodePtr schema = GroupNode::Make("schema", Repetition::REPEATED, nodes);
- descr_.Init(schema);
- return FromParquetSchema(&descr_, column_indices, &result_schema_);
- }
-
- ::arrow::Status ConvertSchema(
- const std::vector<NodePtr>& nodes,
- const std::shared_ptr<const KeyValueMetadata>& key_value_metadata) {
- NodePtr schema = GroupNode::Make("schema", Repetition::REPEATED, nodes);
- descr_.Init(schema);
- return FromParquetSchema(&descr_, {}, key_value_metadata, &result_schema_);
- }
-
- protected:
- SchemaDescriptor descr_;
- std::shared_ptr<::arrow::Schema> result_schema_;
-};
-
-TEST_F(TestConvertParquetSchema, ParquetFlatPrimitives) {
- std::vector<NodePtr> parquet_fields;
- std::vector<std::shared_ptr<Field>> arrow_fields;
-
- parquet_fields.push_back(
- PrimitiveNode::Make("boolean", Repetition::REQUIRED, ParquetType::BOOLEAN));
- arrow_fields.push_back(std::make_shared<Field>("boolean", BOOL, false));
-
- parquet_fields.push_back(
- PrimitiveNode::Make("int32", Repetition::REQUIRED, ParquetType::INT32));
- arrow_fields.push_back(std::make_shared<Field>("int32", INT32, false));
-
- parquet_fields.push_back(
- PrimitiveNode::Make("int64", Repetition::REQUIRED, ParquetType::INT64));
- arrow_fields.push_back(std::make_shared<Field>("int64", INT64, false));
-
- parquet_fields.push_back(PrimitiveNode::Make("timestamp", Repetition::REQUIRED,
- ParquetType::INT64,
- LogicalType::TIMESTAMP_MILLIS));
- arrow_fields.push_back(std::make_shared<Field>("timestamp", TIMESTAMP_MS, false));
-
- parquet_fields.push_back(PrimitiveNode::Make("timestamp[us]", Repetition::REQUIRED,
- ParquetType::INT64,
- LogicalType::TIMESTAMP_MICROS));
- arrow_fields.push_back(std::make_shared<Field>("timestamp[us]", TIMESTAMP_US, false));
-
- parquet_fields.push_back(PrimitiveNode::Make("date", Repetition::REQUIRED,
- ParquetType::INT32, LogicalType::DATE));
- arrow_fields.push_back(std::make_shared<Field>("date", ::arrow::date32(), false));
-
- parquet_fields.push_back(PrimitiveNode::Make(
- "time32", Repetition::REQUIRED, ParquetType::INT32, LogicalType::TIME_MILLIS));
- arrow_fields.push_back(
- std::make_shared<Field>("time32", ::arrow::time32(TimeUnit::MILLI), false));
-
- parquet_fields.push_back(PrimitiveNode::Make(
- "time64", Repetition::REQUIRED, ParquetType::INT64, LogicalType::TIME_MICROS));
- arrow_fields.push_back(
- std::make_shared<Field>("time64", ::arrow::time64(TimeUnit::MICRO), false));
-
- parquet_fields.push_back(
- PrimitiveNode::Make("timestamp96", Repetition::REQUIRED, ParquetType::INT96));
- arrow_fields.push_back(std::make_shared<Field>("timestamp96", TIMESTAMP_NS, false));
-
- parquet_fields.push_back(
- PrimitiveNode::Make("float", Repetition::OPTIONAL, ParquetType::FLOAT));
- arrow_fields.push_back(std::make_shared<Field>("float", FLOAT));
-
- parquet_fields.push_back(
- PrimitiveNode::Make("double", Repetition::OPTIONAL, ParquetType::DOUBLE));
- arrow_fields.push_back(std::make_shared<Field>("double", DOUBLE));
-
- parquet_fields.push_back(
- PrimitiveNode::Make("binary", Repetition::OPTIONAL, ParquetType::BYTE_ARRAY));
- arrow_fields.push_back(std::make_shared<Field>("binary", BINARY));
-
- parquet_fields.push_back(PrimitiveNode::Make(
- "string", Repetition::OPTIONAL, ParquetType::BYTE_ARRAY, LogicalType::UTF8));
- arrow_fields.push_back(std::make_shared<Field>("string", UTF8));
-
- parquet_fields.push_back(PrimitiveNode::Make("flba-binary", Repetition::OPTIONAL,
- ParquetType::FIXED_LEN_BYTE_ARRAY,
- LogicalType::NONE, 12));
- arrow_fields.push_back(
- std::make_shared<Field>("flba-binary", ::arrow::fixed_size_binary(12)));
-
- auto arrow_schema = std::make_shared<::arrow::Schema>(arrow_fields);
- ASSERT_OK(ConvertSchema(parquet_fields));
-
- ASSERT_NO_FATAL_FAILURE(CheckFlatSchema(arrow_schema));
-}
-
-TEST_F(TestConvertParquetSchema, DuplicateFieldNames) {
- std::vector<NodePtr> parquet_fields;
- std::vector<std::shared_ptr<Field>> arrow_fields;
-
- parquet_fields.push_back(
- PrimitiveNode::Make("xxx", Repetition::REQUIRED, ParquetType::BOOLEAN));
- auto arrow_field1 = std::make_shared<Field>("xxx", BOOL, false);
-
- parquet_fields.push_back(
- PrimitiveNode::Make("xxx", Repetition::REQUIRED, ParquetType::INT32));
- auto arrow_field2 = std::make_shared<Field>("xxx", INT32, false);
-
- ASSERT_OK(ConvertSchema(parquet_fields));
- arrow_fields = {arrow_field1, arrow_field2};
- ASSERT_NO_FATAL_FAILURE(
- CheckFlatSchema(std::make_shared<::arrow::Schema>(arrow_fields)));
-
- ASSERT_OK(ConvertSchema(parquet_fields, std::vector<int>({0, 1})));
- arrow_fields = {arrow_field1, arrow_field2};
- ASSERT_NO_FATAL_FAILURE(
- CheckFlatSchema(std::make_shared<::arrow::Schema>(arrow_fields)));
-
- ASSERT_OK(ConvertSchema(parquet_fields, std::vector<int>({1, 0})));
- arrow_fields = {arrow_field2, arrow_field1};
- ASSERT_NO_FATAL_FAILURE(
- CheckFlatSchema(std::make_shared<::arrow::Schema>(arrow_fields)));
-}
-
-TEST_F(TestConvertParquetSchema, ParquetKeyValueMetadata) {
- std::vector<NodePtr> parquet_fields;
- std::vector<std::shared_ptr<Field>> arrow_fields;
-
- parquet_fields.push_back(
- PrimitiveNode::Make("boolean", Repetition::REQUIRED, ParquetType::BOOLEAN));
- arrow_fields.push_back(std::make_shared<Field>("boolean", BOOL, false));
-
- parquet_fields.push_back(
- PrimitiveNode::Make("int32", Repetition::REQUIRED, ParquetType::INT32));
- arrow_fields.push_back(std::make_shared<Field>("int32", INT32, false));
-
- auto key_value_metadata = std::make_shared<KeyValueMetadata>();
- key_value_metadata->Append("foo", "bar");
- key_value_metadata->Append("biz", "baz");
- ASSERT_OK(ConvertSchema(parquet_fields, key_value_metadata));
-
- auto arrow_metadata = result_schema_->metadata();
- ASSERT_EQ("foo", arrow_metadata->key(0));
- ASSERT_EQ("bar", arrow_metadata->value(0));
- ASSERT_EQ("biz", arrow_metadata->key(1));
- ASSERT_EQ("baz", arrow_metadata->value(1));
-}
-
-TEST_F(TestConvertParquetSchema, ParquetEmptyKeyValueMetadata) {
- std::vector<NodePtr> parquet_fields;
- std::vector<std::shared_ptr<Field>> arrow_fields;
-
- parquet_fields.push_back(
- PrimitiveNode::Make("int32", Repetition::REQUIRED, ParquetType::INT32));
- arrow_fields.push_back(std::make_shared<Field>("int32", INT32, false));
-
- std::shared_ptr<KeyValueMetadata> key_value_metadata = nullptr;
- ASSERT_OK(ConvertSchema(parquet_fields, key_value_metadata));
-
- auto arrow_metadata = result_schema_->metadata();
- ASSERT_EQ(arrow_metadata, nullptr);
-}
-
-TEST_F(TestConvertParquetSchema, ParquetFlatDecimals) {
- std::vector<NodePtr> parquet_fields;
- std::vector<std::shared_ptr<Field>> arrow_fields;
-
- parquet_fields.push_back(PrimitiveNode::Make("flba-decimal", Repetition::OPTIONAL,
- ParquetType::FIXED_LEN_BYTE_ARRAY,
- LogicalType::DECIMAL, 4, 8, 4));
- arrow_fields.push_back(std::make_shared<Field>("flba-decimal", DECIMAL_8_4));
-
- parquet_fields.push_back(PrimitiveNode::Make("binary-decimal", Repetition::OPTIONAL,
- ParquetType::BYTE_ARRAY,
- LogicalType::DECIMAL, -1, 8, 4));
- arrow_fields.push_back(std::make_shared<Field>("binary-decimal", DECIMAL_8_4));
-
- parquet_fields.push_back(PrimitiveNode::Make("int32-decimal", Repetition::OPTIONAL,
- ParquetType::INT32, LogicalType::DECIMAL,
- -1, 8, 4));
- arrow_fields.push_back(std::make_shared<Field>("int32-decimal", DECIMAL_8_4));
-
- parquet_fields.push_back(PrimitiveNode::Make("int64-decimal", Repetition::OPTIONAL,
- ParquetType::INT64, LogicalType::DECIMAL,
- -1, 8, 4));
- arrow_fields.push_back(std::make_shared<Field>("int64-decimal", DECIMAL_8_4));
-
- auto arrow_schema = std::make_shared<::arrow::Schema>(arrow_fields);
- ASSERT_OK(ConvertSchema(parquet_fields));
-
- ASSERT_NO_FATAL_FAILURE(CheckFlatSchema(arrow_schema));
-}
-
-TEST_F(TestConvertParquetSchema, ParquetLists) {
- std::vector<NodePtr> parquet_fields;
- std::vector<std::shared_ptr<Field>> arrow_fields;
-
- // LIST encoding example taken from parquet-format/LogicalTypes.md
-
- // // List<String> (list non-null, elements nullable)
- // required group my_list (LIST) {
- // repeated group list {
- // optional binary element (UTF8);
- // }
- // }
- {
- auto element = PrimitiveNode::Make("string", Repetition::OPTIONAL,
- ParquetType::BYTE_ARRAY, LogicalType::UTF8);
- auto list = GroupNode::Make("list", Repetition::REPEATED, {element});
- parquet_fields.push_back(
- GroupNode::Make("my_list", Repetition::REQUIRED, {list}, LogicalType::LIST));
- auto arrow_element = std::make_shared<Field>("string", UTF8, true);
- auto arrow_list = std::make_shared<::arrow::ListType>(arrow_element);
- arrow_fields.push_back(std::make_shared<Field>("my_list", arrow_list, false));
- }
-
- // // List<String> (list nullable, elements non-null)
- // optional group my_list (LIST) {
- // repeated group list {
- // required binary element (UTF8);
- // }
- // }
- {
- auto element = PrimitiveNode::Make("string", Repetition::REQUIRED,
- ParquetType::BYTE_ARRAY, LogicalType::UTF8);
- auto list = GroupNode::Make("list", Repetition::REPEATED, {element});
- parquet_fields.push_back(
- GroupNode::Make("my_list", Repetition::OPTIONAL, {list}, LogicalType::LIST));
- auto arrow_element = std::make_shared<Field>("string", UTF8, false);
- auto arrow_list = std::make_shared<::arrow::ListType>(arrow_element);
- arrow_fields.push_back(std::make_shared<Field>("my_list", arrow_list, true));
- }
-
- // Element types can be nested structures. For example, a list of lists:
- //
- // // List<List<Integer>>
- // optional group array_of_arrays (LIST) {
- // repeated group list {
- // required group element (LIST) {
- // repeated group list {
- // required int32 element;
- // }
- // }
- // }
- // }
- {
- auto inner_element =
- PrimitiveNode::Make("int32", Repetition::REQUIRED, ParquetType::INT32);
- auto inner_list = GroupNode::Make("list", Repetition::REPEATED, {inner_element});
- auto element =
- GroupNode::Make("element", Repetition::REQUIRED, {inner_list}, LogicalType::LIST);
- auto list = GroupNode::Make("list", Repetition::REPEATED, {element});
- parquet_fields.push_back(GroupNode::Make("array_of_arrays", Repetition::OPTIONAL,
- {list}, LogicalType::LIST));
- auto arrow_inner_element = std::make_shared<Field>("int32", INT32, false);
- auto arrow_inner_list = std::make_shared<::arrow::ListType>(arrow_inner_element);
- auto arrow_element = std::make_shared<Field>("element", arrow_inner_list, false);
- auto arrow_list = std::make_shared<::arrow::ListType>(arrow_element);
- arrow_fields.push_back(std::make_shared<Field>("array_of_arrays", arrow_list, true));
- }
-
- // // List<String> (list nullable, elements non-null)
- // optional group my_list (LIST) {
- // repeated group element {
- // required binary str (UTF8);
- // };
- // }
- {
- auto element = PrimitiveNode::Make("str", Repetition::REQUIRED,
- ParquetType::BYTE_ARRAY, LogicalType::UTF8);
- auto list = GroupNode::Make("element", Repetition::REPEATED, {element});
- parquet_fields.push_back(
- GroupNode::Make("my_list", Repetition::OPTIONAL, {list}, LogicalType::LIST));
- auto arrow_element = std::make_shared<Field>("str", UTF8, false);
- auto arrow_list = std::make_shared<::arrow::ListType>(arrow_element);
- arrow_fields.push_back(std::make_shared<Field>("my_list", arrow_list, true));
- }
-
- // // List<Integer> (nullable list, non-null elements)
- // optional group my_list (LIST) {
- // repeated int32 element;
- // }
- {
- auto element =
- PrimitiveNode::Make("element", Repetition::REPEATED, ParquetType::INT32);
- parquet_fields.push_back(
- GroupNode::Make("my_list", Repetition::OPTIONAL, {element}, LogicalType::LIST));
- auto arrow_element = std::make_shared<Field>("element", INT32, false);
- auto arrow_list = std::make_shared<::arrow::ListType>(arrow_element);
- arrow_fields.push_back(std::make_shared<Field>("my_list", arrow_list, true));
- }
-
- // // List<Tuple<String, Integer>> (nullable list, non-null elements)
- // optional group my_list (LIST) {
- // repeated group element {
- // required binary str (UTF8);
- // required int32 num;
- // };
- // }
- {
- auto str_element = PrimitiveNode::Make("str", Repetition::REQUIRED,
- ParquetType::BYTE_ARRAY, LogicalType::UTF8);
- auto num_element =
- PrimitiveNode::Make("num", Repetition::REQUIRED, ParquetType::INT32);
- auto element =
- GroupNode::Make("element", Repetition::REPEATED, {str_element, num_element});
- parquet_fields.push_back(
- GroupNode::Make("my_list", Repetition::OPTIONAL, {element}, LogicalType::LIST));
- auto arrow_str = std::make_shared<Field>("str", UTF8, false);
- auto arrow_num = std::make_shared<Field>("num", INT32, false);
- std::vector<std::shared_ptr<Field>> fields({arrow_str, arrow_num});
- auto arrow_struct = std::make_shared<::arrow::StructType>(fields);
- auto arrow_element = std::make_shared<Field>("element", arrow_struct, false);
- auto arrow_list = std::make_shared<::arrow::ListType>(arrow_element);
- arrow_fields.push_back(std::make_shared<Field>("my_list", arrow_list, true));
- }
-
- // // List<OneTuple<String>> (nullable list, non-null elements)
- // optional group my_list (LIST) {
- // repeated group array {
- // required binary str (UTF8);
- // };
- // }
- // Special case: group is named array
- {
- auto element = PrimitiveNode::Make("str", Repetition::REQUIRED,
- ParquetType::BYTE_ARRAY, LogicalType::UTF8);
- auto array = GroupNode::Make("array", Repetition::REPEATED, {element});
- parquet_fields.push_back(
- GroupNode::Make("my_list", Repetition::OPTIONAL, {array}, LogicalType::LIST));
- auto arrow_str = std::make_shared<Field>("str", UTF8, false);
- std::vector<std::shared_ptr<Field>> fields({arrow_str});
- auto arrow_struct = std::make_shared<::arrow::StructType>(fields);
- auto arrow_element = std::make_shared<Field>("array", arrow_struct, false);
- auto arrow_list = std::make_shared<::arrow::ListType>(arrow_element);
- arrow_fields.push_back(std::make_shared<Field>("my_list", arrow_list, true));
- }
-
- // // List<OneTuple<String>> (nullable list, non-null elements)
- // optional group my_list (LIST) {
- // repeated group my_list_tuple {
- // required binary str (UTF8);
- // };
- // }
- // Special case: group named ends in _tuple
- {
- auto element = PrimitiveNode::Make("str", Repetition::REQUIRED,
- ParquetType::BYTE_ARRAY, LogicalType::UTF8);
- auto array = GroupNode::Make("my_list_tuple", Repetition::REPEATED, {element});
- parquet_fields.push_back(
- GroupNode::Make("my_list", Repetition::OPTIONAL, {array}, LogicalType::LIST));
- auto arrow_str = std::make_shared<Field>("str", UTF8, false);
- std::vector<std::shared_ptr<Field>> fields({arrow_str});
- auto arrow_struct = std::make_shared<::arrow::StructType>(fields);
- auto arrow_element = std::make_shared<Field>("my_list_tuple", arrow_struct, false);
- auto arrow_list = std::make_shared<::arrow::ListType>(arrow_element);
- arrow_fields.push_back(std::make_shared<Field>("my_list", arrow_list, true));
- }
-
- // One-level encoding: Only allows required lists with required cells
- // repeated value_type name
- {
- parquet_fields.push_back(
- PrimitiveNode::Make("name", Repetition::REPEATED, ParquetType::INT32));
- auto arrow_element = std::make_shared<Field>("name", INT32, false);
- auto arrow_list = std::make_shared<::arrow::ListType>(arrow_element);
- arrow_fields.push_back(std::make_shared<Field>("name", arrow_list, false));
- }
-
- auto arrow_schema = std::make_shared<::arrow::Schema>(arrow_fields);
- ASSERT_OK(ConvertSchema(parquet_fields));
-
- ASSERT_NO_FATAL_FAILURE(CheckFlatSchema(arrow_schema));
-}
-
-TEST_F(TestConvertParquetSchema, UnsupportedThings) {
- std::vector<NodePtr> unsupported_nodes;
-
- for (const NodePtr& node : unsupported_nodes) {
- ASSERT_RAISES(NotImplemented, ConvertSchema({node}));
- }
-}
-
-TEST_F(TestConvertParquetSchema, ParquetNestedSchema) {
- std::vector<NodePtr> parquet_fields;
- std::vector<std::shared_ptr<Field>> arrow_fields;
-
- // required group group1 {
- // required bool leaf1;
- // required int32 leaf2;
- // }
- // required int64 leaf3;
- {
- parquet_fields.push_back(GroupNode::Make(
- "group1", Repetition::REQUIRED,
- {PrimitiveNode::Make("leaf1", Repetition::REQUIRED, ParquetType::BOOLEAN),
- PrimitiveNode::Make("leaf2", Repetition::REQUIRED, ParquetType::INT32)}));
- parquet_fields.push_back(
- PrimitiveNode::Make("leaf3", Repetition::REQUIRED, ParquetType::INT64));
-
- auto group1_fields = {std::make_shared<Field>("leaf1", BOOL, false),
- std::make_shared<Field>("leaf2", INT32, false)};
- auto arrow_group1_type = std::make_shared<::arrow::StructType>(group1_fields);
- arrow_fields.push_back(std::make_shared<Field>("group1", arrow_group1_type, false));
- arrow_fields.push_back(std::make_shared<Field>("leaf3", INT64, false));
- }
-
- auto arrow_schema = std::make_shared<::arrow::Schema>(arrow_fields);
- ASSERT_OK(ConvertSchema(parquet_fields));
-
- ASSERT_NO_FATAL_FAILURE(CheckFlatSchema(arrow_schema));
-}
-
-TEST_F(TestConvertParquetSchema, ParquetNestedSchemaPartial) {
- std::vector<NodePtr> parquet_fields;
- std::vector<std::shared_ptr<Field>> arrow_fields;
-
- // Full Parquet Schema:
- // required group group1 {
- // required int64 leaf1;
- // required int64 leaf2;
- // }
- // required group group2 {
- // required int64 leaf3;
- // required int64 leaf4;
- // }
- // required int64 leaf5;
- //
- // Expected partial arrow schema (columns 0, 3, 4):
- // required group group1 {
- // required int64 leaf1;
- // }
- // required group group2 {
- // required int64 leaf4;
- // }
- // required int64 leaf5;
- {
- parquet_fields.push_back(GroupNode::Make(
- "group1", Repetition::REQUIRED,
- {PrimitiveNode::Make("leaf1", Repetition::REQUIRED, ParquetType::INT64),
- PrimitiveNode::Make("leaf2", Repetition::REQUIRED, ParquetType::INT64)}));
- parquet_fields.push_back(GroupNode::Make(
- "group2", Repetition::REQUIRED,
- {PrimitiveNode::Make("leaf3", Repetition::REQUIRED, ParquetType::INT64),
- PrimitiveNode::Make("leaf4", Repetition::REQUIRED, ParquetType::INT64)}));
- parquet_fields.push_back(
- PrimitiveNode::Make("leaf5", Repetition::REQUIRED, ParquetType::INT64));
-
- auto group1_fields = {std::make_shared<Field>("leaf1", INT64, false)};
- auto arrow_group1_type = std::make_shared<::arrow::StructType>(group1_fields);
- auto group2_fields = {std::make_shared<Field>("leaf4", INT64, false)};
- auto arrow_group2_type = std::make_shared<::arrow::StructType>(group2_fields);
-
- arrow_fields.push_back(std::make_shared<Field>("group1", arrow_group1_type, false));
- arrow_fields.push_back(std::make_shared<Field>("group2", arrow_group2_type, false));
- arrow_fields.push_back(std::make_shared<Field>("leaf5", INT64, false));
- }
-
- auto arrow_schema = std::make_shared<::arrow::Schema>(arrow_fields);
- ASSERT_OK(ConvertSchema(parquet_fields, std::vector<int>{0, 3, 4}));
-
- ASSERT_NO_FATAL_FAILURE(CheckFlatSchema(arrow_schema));
-}
-
-TEST_F(TestConvertParquetSchema, ParquetNestedSchemaPartialOrdering) {
- std::vector<NodePtr> parquet_fields;
- std::vector<std::shared_ptr<Field>> arrow_fields;
-
- // Full Parquet Schema:
- // required group group1 {
- // required int64 leaf1;
- // required int64 leaf2;
- // }
- // required group group2 {
- // required int64 leaf3;
- // required int64 leaf4;
- // }
- // required int64 leaf5;
- //
- // Expected partial arrow schema (columns 3, 4, 0):
- // required group group2 {
- // required int64 leaf4;
- // }
- // required int64 leaf5;
- // required group group1 {
- // required int64 leaf1;
- // }
- {
- parquet_fields.push_back(GroupNode::Make(
- "group1", Repetition::REQUIRED,
- {PrimitiveNode::Make("leaf1", Repetition::REQUIRED, ParquetType::INT64),
- PrimitiveNode::Make("leaf2", Repetition::REQUIRED, ParquetType::INT64)}));
- parquet_fields.push_back(GroupNode::Make(
- "group2", Repetition::REQUIRED,
- {PrimitiveNode::Make("leaf3", Repetition::REQUIRED, ParquetType::INT64),
- PrimitiveNode::Make("leaf4", Repetition::REQUIRED, ParquetType::INT64)}));
- parquet_fields.push_back(
- PrimitiveNode::Make("leaf5", Repetition::REQUIRED, ParquetType::INT64));
-
- auto group1_fields = {std::make_shared<Field>("leaf1", INT64, false)};
- auto arrow_group1_type = std::make_shared<::arrow::StructType>(group1_fields);
- auto group2_fields = {std::make_shared<Field>("leaf4", INT64, false)};
- auto arrow_group2_type = std::make_shared<::arrow::StructType>(group2_fields);
-
- arrow_fields.push_back(std::make_shared<Field>("group2", arrow_group2_type, false));
- arrow_fields.push_back(std::make_shared<Field>("leaf5", INT64, false));
- arrow_fields.push_back(std::make_shared<Field>("group1", arrow_group1_type, false));
- }
-
- auto arrow_schema = std::make_shared<::arrow::Schema>(arrow_fields);
- ASSERT_OK(ConvertSchema(parquet_fields, std::vector<int>{3, 4, 0}));
-
- ASSERT_NO_FATAL_FAILURE(CheckFlatSchema(arrow_schema));
-}
-TEST_F(TestConvertParquetSchema, ParquetRepeatedNestedSchema) {
- std::vector<NodePtr> parquet_fields;
- std::vector<std::shared_ptr<Field>> arrow_fields;
- {
- // optional int32 leaf1;
- // repeated group outerGroup {
- // optional int32 leaf2;
- // repeated group innerGroup {
- // optional int32 leaf3;
- // }
- // }
- parquet_fields.push_back(
- PrimitiveNode::Make("leaf1", Repetition::OPTIONAL, ParquetType::INT32));
- parquet_fields.push_back(GroupNode::Make(
- "outerGroup", Repetition::REPEATED,
- {PrimitiveNode::Make("leaf2", Repetition::OPTIONAL, ParquetType::INT32),
- GroupNode::Make(
- "innerGroup", Repetition::REPEATED,
- {PrimitiveNode::Make("leaf3", Repetition::OPTIONAL, ParquetType::INT32)})}));
-
- auto inner_group_fields = {std::make_shared<Field>("leaf3", INT32, true)};
- auto inner_group_type = std::make_shared<::arrow::StructType>(inner_group_fields);
- auto outer_group_fields = {
- std::make_shared<Field>("leaf2", INT32, true),
- std::make_shared<Field>(
- "innerGroup",
- ::arrow::list(std::make_shared<Field>("innerGroup", inner_group_type, false)),
- false)};
- auto outer_group_type = std::make_shared<::arrow::StructType>(outer_group_fields);
-
- arrow_fields.push_back(std::make_shared<Field>("leaf1", INT32, true));
- arrow_fields.push_back(std::make_shared<Field>(
- "outerGroup",
- ::arrow::list(std::make_shared<Field>("outerGroup", outer_group_type, false)),
- false));
- }
- auto arrow_schema = std::make_shared<::arrow::Schema>(arrow_fields);
- ASSERT_OK(ConvertSchema(parquet_fields));
-
- ASSERT_NO_FATAL_FAILURE(CheckFlatSchema(arrow_schema));
-}
-
-class TestConvertArrowSchema : public ::testing::Test {
- public:
- virtual void SetUp() {}
-
- void CheckFlatSchema(const std::vector<NodePtr>& nodes) {
- NodePtr schema_node = GroupNode::Make("schema", Repetition::REPEATED, nodes);
- const GroupNode* expected_schema_node =
- static_cast<const GroupNode*>(schema_node.get());
- const GroupNode* result_schema_node = result_schema_->group_node();
-
- ASSERT_EQ(expected_schema_node->field_count(), result_schema_node->field_count());
-
- for (int i = 0; i < expected_schema_node->field_count(); i++) {
- auto lhs = result_schema_node->field(i);
- auto rhs = expected_schema_node->field(i);
- EXPECT_TRUE(lhs->Equals(rhs.get()));
- }
- }
-
- ::arrow::Status ConvertSchema(const std::vector<std::shared_ptr<Field>>& fields) {
- arrow_schema_ = std::make_shared<::arrow::Schema>(fields);
- std::shared_ptr<::parquet::WriterProperties> properties =
- ::parquet::default_writer_properties();
- return ToParquetSchema(arrow_schema_.get(), *properties.get(), &result_schema_);
- }
-
- protected:
- std::shared_ptr<::arrow::Schema> arrow_schema_;
- std::shared_ptr<SchemaDescriptor> result_schema_;
-};
-
-TEST_F(TestConvertArrowSchema, ParquetFlatPrimitives) {
- std::vector<NodePtr> parquet_fields;
- std::vector<std::shared_ptr<Field>> arrow_fields;
-
- parquet_fields.push_back(
- PrimitiveNode::Make("boolean", Repetition::REQUIRED, ParquetType::BOOLEAN));
- arrow_fields.push_back(std::make_shared<Field>("boolean", BOOL, false));
-
- parquet_fields.push_back(
- PrimitiveNode::Make("int32", Repetition::REQUIRED, ParquetType::INT32));
- arrow_fields.push_back(std::make_shared<Field>("int32", INT32, false));
-
- parquet_fields.push_back(
- PrimitiveNode::Make("int64", Repetition::REQUIRED, ParquetType::INT64));
- arrow_fields.push_back(std::make_shared<Field>("int64", INT64, false));
-
- parquet_fields.push_back(PrimitiveNode::Make("date", Repetition::REQUIRED,
- ParquetType::INT32, LogicalType::DATE));
- arrow_fields.push_back(std::make_shared<Field>("date", ::arrow::date32(), false));
-
- parquet_fields.push_back(PrimitiveNode::Make("date64", Repetition::REQUIRED,
- ParquetType::INT32, LogicalType::DATE));
- arrow_fields.push_back(std::make_shared<Field>("date64", ::arrow::date64(), false));
-
- parquet_fields.push_back(PrimitiveNode::Make("timestamp", Repetition::REQUIRED,
- ParquetType::INT64,
- LogicalType::TIMESTAMP_MILLIS));
- arrow_fields.push_back(std::make_shared<Field>("timestamp", TIMESTAMP_MS, false));
-
- parquet_fields.push_back(PrimitiveNode::Make("timestamp[us]", Repetition::REQUIRED,
- ParquetType::INT64,
- LogicalType::TIMESTAMP_MICROS));
- arrow_fields.push_back(std::make_shared<Field>("timestamp[us]", TIMESTAMP_US, false));
-
- parquet_fields.push_back(
- PrimitiveNode::Make("float", Repetition::OPTIONAL, ParquetType::FLOAT));
- arrow_fields.push_back(std::make_shared<Field>("float", FLOAT));
-
- parquet_fields.push_back(
- PrimitiveNode::Make("double", Repetition::OPTIONAL, ParquetType::DOUBLE));
- arrow_fields.push_back(std::make_shared<Field>("double", DOUBLE));
-
- parquet_fields.push_back(PrimitiveNode::Make(
- "string", Repetition::OPTIONAL, ParquetType::BYTE_ARRAY, LogicalType::UTF8));
- arrow_fields.push_back(std::make_shared<Field>("string", UTF8));
-
- parquet_fields.push_back(PrimitiveNode::Make(
- "binary", Repetition::OPTIONAL, ParquetType::BYTE_ARRAY, LogicalType::NONE));
- arrow_fields.push_back(std::make_shared<Field>("binary", BINARY));
-
- ASSERT_OK(ConvertSchema(arrow_fields));
-
- ASSERT_NO_FATAL_FAILURE(CheckFlatSchema(parquet_fields));
-}
-
-TEST_F(TestConvertArrowSchema, ParquetFlatPrimitivesAsDictionaries) {
- std::vector<NodePtr> parquet_fields;
- std::vector<std::shared_ptr<Field>> arrow_fields;
- std::shared_ptr<::arrow::Array> dict;
-
- parquet_fields.push_back(
- PrimitiveNode::Make("int32", Repetition::REQUIRED, ParquetType::INT32));
- ArrayFromVector<::arrow::Int32Type, int32_t>(std::vector<int32_t>(), &dict);
- arrow_fields.push_back(
- ::arrow::field("int32", ::arrow::dictionary(::arrow::int8(), dict), false));
-
- parquet_fields.push_back(
- PrimitiveNode::Make("int64", Repetition::REQUIRED, ParquetType::INT64));
- ArrayFromVector<::arrow::Int64Type, int64_t>(std::vector<int64_t>(), &dict);
- arrow_fields.push_back(std::make_shared<Field>(
- "int64", ::arrow::dictionary(::arrow::int8(), dict), false));
-
- parquet_fields.push_back(PrimitiveNode::Make("date", Repetition::REQUIRED,
- ParquetType::INT32, LogicalType::DATE));
- ArrayFromVector<::arrow::Date32Type, int32_t>(std::vector<int32_t>(), &dict);
- arrow_fields.push_back(
- std::make_shared<Field>("date", ::arrow::dictionary(::arrow::int8(), dict), false));
-
- parquet_fields.push_back(PrimitiveNode::Make("date64", Repetition::REQUIRED,
- ParquetType::INT32, LogicalType::DATE));
- ArrayFromVector<::arrow::Date64Type, int64_t>(std::vector<int64_t>(), &dict);
- arrow_fields.push_back(std::make_shared<Field>(
- "date64", ::arrow::dictionary(::arrow::int8(), dict), false));
-
- parquet_fields.push_back(
- PrimitiveNode::Make("float", Repetition::OPTIONAL, ParquetType::FLOAT));
- ArrayFromVector<::arrow::FloatType, float>(std::vector<float>(), &dict);
- arrow_fields.push_back(
- std::make_shared<Field>("float", ::arrow::dictionary(::arrow::int8(), dict)));
-
- parquet_fields.push_back(
- PrimitiveNode::Make("double", Repetition::OPTIONAL, ParquetType::DOUBLE));
- ArrayFromVector<::arrow::DoubleType, double>(std::vector<double>(), &dict);
- arrow_fields.push_back(
- std::make_shared<Field>("double", ::arrow::dictionary(::arrow::int8(), dict)));
-
- parquet_fields.push_back(PrimitiveNode::Make(
- "string", Repetition::OPTIONAL, ParquetType::BYTE_ARRAY, LogicalType::UTF8));
- ::arrow::StringBuilder string_builder(::arrow::default_memory_pool());
- ASSERT_OK(string_builder.Finish(&dict));
- arrow_fields.push_back(
- std::make_shared<Field>("string", ::arrow::dictionary(::arrow::int8(), dict)));
-
- parquet_fields.push_back(PrimitiveNode::Make(
- "binary", Repetition::OPTIONAL, ParquetType::BYTE_ARRAY, LogicalType::NONE));
- ::arrow::BinaryBuilder binary_builder(::arrow::default_memory_pool());
- ASSERT_OK(binary_builder.Finish(&dict));
- arrow_fields.push_back(
- std::make_shared<Field>("binary", ::arrow::dictionary(::arrow::int8(), dict)));
-
- ASSERT_OK(ConvertSchema(arrow_fields));
-
- ASSERT_NO_FATAL_FAILURE(CheckFlatSchema(parquet_fields));
-}
-
-TEST_F(TestConvertArrowSchema, ParquetLists) {
- std::vector<NodePtr> parquet_fields;
- std::vector<std::shared_ptr<Field>> arrow_fields;
-
- // parquet_arrow will always generate 3-level LIST encodings
-
- // // List<String> (list non-null, elements nullable)
- // required group my_list (LIST) {
- // repeated group list {
- // optional binary element (UTF8);
- // }
- // }
- {
- auto element = PrimitiveNode::Make("string", Repetition::OPTIONAL,
- ParquetType::BYTE_ARRAY, LogicalType::UTF8);
- auto list = GroupNode::Make("list", Repetition::REPEATED, {element});
- parquet_fields.push_back(
- GroupNode::Make("my_list", Repetition::REQUIRED, {list}, LogicalType::LIST));
- auto arrow_element = std::make_shared<Field>("string", UTF8, true);
- auto arrow_list = std::make_shared<::arrow::ListType>(arrow_element);
- arrow_fields.push_back(std::make_shared<Field>("my_list", arrow_list, false));
- }
-
- // // List<String> (list nullable, elements non-null)
- // optional group my_list (LIST) {
- // repeated group list {
- // required binary element (UTF8);
- // }
- // }
- {
- auto element = PrimitiveNode::Make("string", Repetition::REQUIRED,
- ParquetType::BYTE_ARRAY, LogicalType::UTF8);
- auto list = GroupNode::Make("list", Repetition::REPEATED, {element});
- parquet_fields.push_back(
- GroupNode::Make("my_list", Repetition::OPTIONAL, {list}, LogicalType::LIST));
- auto arrow_element = std::make_shared<Field>("string", UTF8, false);
- auto arrow_list = std::make_shared<::arrow::ListType>(arrow_element);
- arrow_fields.push_back(std::make_shared<Field>("my_list", arrow_list, true));
- }
-
- ASSERT_OK(ConvertSchema(arrow_fields));
-
- ASSERT_NO_FATAL_FAILURE(CheckFlatSchema(parquet_fields));
-}
-
-TEST_F(TestConvertArrowSchema, UnsupportedTypes) {
- std::vector<std::shared_ptr<Field>> unsupported_fields = {
- ::arrow::field("f0", ::arrow::time64(TimeUnit::NANO))};
-
- for (const auto& field : unsupported_fields) {
- ASSERT_RAISES(NotImplemented, ConvertSchema({field}));
- }
-}
-
-TEST_F(TestConvertArrowSchema, ParquetFlatDecimals) {
- std::vector<NodePtr> parquet_fields;
- std::vector<std::shared_ptr<Field>> arrow_fields;
-
- // TODO: Test Decimal Arrow -> Parquet conversion
-
- ASSERT_OK(ConvertSchema(arrow_fields));
-
- ASSERT_NO_FATAL_FAILURE(CheckFlatSchema(parquet_fields));
-}
-
-TEST(InvalidSchema, ParquetNegativeDecimalScale) {
- const auto& type = ::arrow::decimal(23, -2);
- const auto& field = ::arrow::field("f0", type);
- const auto& arrow_schema = ::arrow::schema({field});
- std::shared_ptr<::parquet::WriterProperties> properties =
- ::parquet::default_writer_properties();
- std::shared_ptr<SchemaDescriptor> result_schema;
-
- ASSERT_RAISES(IOError,
- ToParquetSchema(arrow_schema.get(), *properties.get(), &result_schema));
-}
-
-} // namespace arrow
-} // namespace parquet
diff --git a/src/parquet/arrow/reader.cc b/src/parquet/arrow/reader.cc
deleted file mode 100644
index 2e4dc81..0000000
--- a/src/parquet/arrow/reader.cc
+++ /dev/null
@@ -1,1558 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#include "parquet/arrow/reader.h"
-
-#include <algorithm>
-#include <atomic>
-#include <chrono>
-#include <mutex>
-#include <queue>
-#include <string>
-#include <thread>
-#include <type_traits>
-#include <vector>
-
-#include "arrow/api.h"
-#include "arrow/util/bit-util.h"
-#include "arrow/util/decimal.h"
-#include "arrow/util/logging.h"
-#include "arrow/util/thread-pool.h"
-
-#include "parquet/arrow/record_reader.h"
-#include "parquet/arrow/schema.h"
-#include "parquet/column_reader.h"
-#include "parquet/schema.h"
-#include "parquet/util/schema-util.h"
-
-using arrow::Array;
-using arrow::BooleanArray;
-using arrow::Column;
-using arrow::Field;
-using arrow::Int32Array;
-using arrow::ListArray;
-using arrow::MemoryPool;
-using arrow::ResizableBuffer;
-using arrow::Status;
-using arrow::StructArray;
-using arrow::Table;
-using arrow::TimestampArray;
-
-using parquet::schema::Node;
-
-// Help reduce verbosity
-using ParquetReader = parquet::ParquetFileReader;
-using arrow::RecordBatchReader;
-
-using parquet::internal::RecordReader;
-
-namespace parquet {
-namespace arrow {
-
-using ::arrow::BitUtil::BytesForBits;
-
-constexpr int64_t kJulianToUnixEpochDays = 2440588LL;
-constexpr int64_t kMillisecondsInADay = 86400000LL;
-constexpr int64_t kNanosecondsInADay = kMillisecondsInADay * 1000LL * 1000LL;
-
-static inline int64_t impala_timestamp_to_nanoseconds(const Int96& impala_timestamp) {
- int64_t days_since_epoch = impala_timestamp.value[2] - kJulianToUnixEpochDays;
- int64_t nanoseconds = *(reinterpret_cast<const int64_t*>(&(impala_timestamp.value)));
- return days_since_epoch * kNanosecondsInADay + nanoseconds;
-}
-
-template <typename ArrowType>
-using ArrayType = typename ::arrow::TypeTraits<ArrowType>::ArrayType;
-
-// ----------------------------------------------------------------------
-// Iteration utilities
-
-// Abstraction to decouple row group iteration details from the ColumnReader,
-// so we can read only a single row group if we want
-class FileColumnIterator {
- public:
- explicit FileColumnIterator(int column_index, ParquetFileReader* reader)
- : column_index_(column_index),
- reader_(reader),
- schema_(reader->metadata()->schema()) {}
-
- virtual ~FileColumnIterator() {}
-
- virtual std::unique_ptr<::parquet::PageReader> NextChunk() = 0;
-
- const SchemaDescriptor* schema() const { return schema_; }
-
- const ColumnDescriptor* descr() const { return schema_->Column(column_index_); }
-
- std::shared_ptr<FileMetaData> metadata() const { return reader_->metadata(); }
-
- int column_index() const { return column_index_; }
-
- protected:
- int column_index_;
- ParquetFileReader* reader_;
- const SchemaDescriptor* schema_;
-};
-
-class AllRowGroupsIterator : public FileColumnIterator {
- public:
- explicit AllRowGroupsIterator(int column_index, ParquetFileReader* reader)
- : FileColumnIterator(column_index, reader), next_row_group_(0) {}
-
- std::unique_ptr<::parquet::PageReader> NextChunk() override {
- std::unique_ptr<::parquet::PageReader> result;
- if (next_row_group_ < reader_->metadata()->num_row_groups()) {
- result = reader_->RowGroup(next_row_group_)->GetColumnPageReader(column_index_);
- next_row_group_++;
- } else {
- result = nullptr;
- }
- return result;
- }
-
- private:
- int next_row_group_;
-};
-
-class SingleRowGroupIterator : public FileColumnIterator {
- public:
- explicit SingleRowGroupIterator(int column_index, int row_group_number,
- ParquetFileReader* reader)
- : FileColumnIterator(column_index, reader),
- row_group_number_(row_group_number),
- done_(false) {}
-
- std::unique_ptr<::parquet::PageReader> NextChunk() override {
- if (done_) {
- return nullptr;
- }
-
- auto result =
- reader_->RowGroup(row_group_number_)->GetColumnPageReader(column_index_);
- done_ = true;
- return result;
- }
-
- private:
- int row_group_number_;
- bool done_;
-};
-
-class RowGroupRecordBatchReader : public ::arrow::RecordBatchReader {
- public:
- explicit RowGroupRecordBatchReader(const std::vector<int>& row_group_indices,
- const std::vector<int>& column_indices,
- std::shared_ptr<::arrow::Schema> schema,
- FileReader* reader)
- : row_group_indices_(row_group_indices),
- column_indices_(column_indices),
- schema_(schema),
- file_reader_(reader),
- next_row_group_(0) {}
-
- ~RowGroupRecordBatchReader() override {}
-
- std::shared_ptr<::arrow::Schema> schema() const override { return schema_; }
-
- Status ReadNext(std::shared_ptr<::arrow::RecordBatch>* out) override {
- if (table_ != nullptr) { // one row group has been loaded
- std::shared_ptr<::arrow::RecordBatch> tmp;
- RETURN_NOT_OK(table_batch_reader_->ReadNext(&tmp));
- if (tmp != nullptr) { // some column chunks are left in table
- *out = tmp;
- return Status::OK();
- } else { // the entire table is consumed
- table_batch_reader_.reset();
- table_.reset();
- }
- }
-
- // all row groups has been consumed
- if (next_row_group_ == row_group_indices_.size()) {
- *out = nullptr;
- return Status::OK();
- }
-
- RETURN_NOT_OK(file_reader_->ReadRowGroup(row_group_indices_[next_row_group_],
- column_indices_, &table_));
-
- next_row_group_++;
- table_batch_reader_.reset(new ::arrow::TableBatchReader(*table_.get()));
- return table_batch_reader_->ReadNext(out);
- }
-
- private:
- std::vector<int> row_group_indices_;
- std::vector<int> column_indices_;
- std::shared_ptr<::arrow::Schema> schema_;
- FileReader* file_reader_;
- size_t next_row_group_;
- std::shared_ptr<::arrow::Table> table_;
- std::unique_ptr<::arrow::TableBatchReader> table_batch_reader_;
-};
-
-// ----------------------------------------------------------------------
-// File reader implementation
-
-class FileReader::Impl {
- public:
- Impl(MemoryPool* pool, std::unique_ptr<ParquetFileReader> reader)
- : pool_(pool), reader_(std::move(reader)), use_threads_(false) {}
-
- virtual ~Impl() {}
-
- Status GetColumn(int i, std::unique_ptr<ColumnReader>* out);
- Status ReadSchemaField(int i, std::shared_ptr<Array>* out);
- Status ReadSchemaField(int i, const std::vector<int>& indices,
- std::shared_ptr<Array>* out);
- Status GetReaderForNode(int index, const Node* node, const std::vector<int>& indices,
- int16_t def_level,
- std::unique_ptr<ColumnReader::ColumnReaderImpl>* out);
- Status ReadColumn(int i, std::shared_ptr<Array>* out);
- Status ReadColumnChunk(int column_index, int row_group_index,
- std::shared_ptr<Array>* out);
- Status GetSchema(std::shared_ptr<::arrow::Schema>* out);
- Status GetSchema(const std::vector<int>& indices,
- std::shared_ptr<::arrow::Schema>* out);
- Status ReadRowGroup(int row_group_index, std::shared_ptr<Table>* table);
- Status ReadRowGroup(int row_group_index, const std::vector<int>& indices,
- std::shared_ptr<::arrow::Table>* out);
- Status ReadTable(const std::vector<int>& indices, std::shared_ptr<Table>* table);
- Status ReadTable(std::shared_ptr<Table>* table);
- Status ReadRowGroups(const std::vector<int>& row_groups, std::shared_ptr<Table>* table);
- Status ReadRowGroups(const std::vector<int>& row_groups,
- const std::vector<int>& indices,
- std::shared_ptr<::arrow::Table>* out);
-
- bool CheckForFlatColumn(const ColumnDescriptor* descr);
- bool CheckForFlatListColumn(const ColumnDescriptor* descr);
-
- const ParquetFileReader* parquet_reader() const { return reader_.get(); }
-
- int num_row_groups() const { return reader_->metadata()->num_row_groups(); }
-
- int num_columns() const { return reader_->metadata()->num_columns(); }
-
- void set_use_threads(bool use_threads) { use_threads_ = use_threads; }
-
- ParquetFileReader* reader() { return reader_.get(); }
-
- private:
- MemoryPool* pool_;
- std::unique_ptr<ParquetFileReader> reader_;
- bool use_threads_;
-};
-
-class ColumnReader::ColumnReaderImpl {
- public:
- virtual ~ColumnReaderImpl() {}
- virtual Status NextBatch(int64_t records_to_read, std::shared_ptr<Array>* out) = 0;
- virtual Status GetDefLevels(const int16_t** data, size_t* length) = 0;
- virtual Status GetRepLevels(const int16_t** data, size_t* length) = 0;
- virtual const std::shared_ptr<Field> field() = 0;
-};
-
-// Reader implementation for primitive arrays
-class PARQUET_NO_EXPORT PrimitiveImpl : public ColumnReader::ColumnReaderImpl {
- public:
- PrimitiveImpl(MemoryPool* pool, std::unique_ptr<FileColumnIterator> input)
- : pool_(pool), input_(std::move(input)), descr_(input_->descr()) {
- record_reader_ = RecordReader::Make(descr_, pool_);
- DCHECK(NodeToField(*input_->descr()->schema_node(), &field_).ok());
- NextRowGroup();
- }
-
- Status NextBatch(int64_t records_to_read, std::shared_ptr<Array>* out) override;
-
- template <typename ParquetType>
- Status WrapIntoListArray(std::shared_ptr<Array>* array);
-
- Status GetDefLevels(const int16_t** data, size_t* length) override;
- Status GetRepLevels(const int16_t** data, size_t* length) override;
-
- const std::shared_ptr<Field> field() override { return field_; }
-
- private:
- void NextRowGroup();
-
- MemoryPool* pool_;
- std::unique_ptr<FileColumnIterator> input_;
- const ColumnDescriptor* descr_;
-
- std::shared_ptr<RecordReader> record_reader_;
-
- std::shared_ptr<Field> field_;
-};
-
-// Reader implementation for struct array
-class PARQUET_NO_EXPORT StructImpl : public ColumnReader::ColumnReaderImpl {
- public:
- explicit StructImpl(const std::vector<std::shared_ptr<ColumnReaderImpl>>& children,
- int16_t struct_def_level, MemoryPool* pool, const Node* node)
- : children_(children), struct_def_level_(struct_def_level), pool_(pool) {
- InitField(node, children);
- }
-
- Status NextBatch(int64_t records_to_read, std::shared_ptr<Array>* out) override;
- Status GetDefLevels(const int16_t** data, size_t* length) override;
- Status GetRepLevels(const int16_t** data, size_t* length) override;
- const std::shared_ptr<Field> field() override { return field_; }
-
- private:
- std::vector<std::shared_ptr<ColumnReaderImpl>> children_;
- int16_t struct_def_level_;
- MemoryPool* pool_;
- std::shared_ptr<Field> field_;
- std::shared_ptr<ResizableBuffer> def_levels_buffer_;
-
- Status DefLevelsToNullArray(std::shared_ptr<Buffer>* null_bitmap, int64_t* null_count);
- void InitField(const Node* node,
- const std::vector<std::shared_ptr<ColumnReaderImpl>>& children);
-};
-
-FileReader::FileReader(MemoryPool* pool, std::unique_ptr<ParquetFileReader> reader)
- : impl_(new FileReader::Impl(pool, std::move(reader))) {}
-
-FileReader::~FileReader() {}
-
-Status FileReader::Impl::GetColumn(int i, std::unique_ptr<ColumnReader>* out) {
- std::unique_ptr<FileColumnIterator> input(new AllRowGroupsIterator(i, reader_.get()));
-
- std::unique_ptr<ColumnReader::ColumnReaderImpl> impl(
- new PrimitiveImpl(pool_, std::move(input)));
- *out = std::unique_ptr<ColumnReader>(new ColumnReader(std::move(impl)));
- return Status::OK();
-}
-
-Status FileReader::Impl::GetReaderForNode(
- int index, const Node* node, const std::vector<int>& indices, int16_t def_level,
- std::unique_ptr<ColumnReader::ColumnReaderImpl>* out) {
- *out = nullptr;
-
- if (IsSimpleStruct(node)) {
- const schema::GroupNode* group = static_cast<const schema::GroupNode*>(node);
- std::vector<std::shared_ptr<ColumnReader::ColumnReaderImpl>> children;
- for (int i = 0; i < group->field_count(); i++) {
- std::unique_ptr<ColumnReader::ColumnReaderImpl> child_reader;
- // TODO(itaiin): Remove the -1 index hack when all types of nested reads
- // are supported. This currently just signals the lower level reader resolution
- // to abort
- RETURN_NOT_OK(GetReaderForNode(index, group->field(i).get(), indices,
- static_cast<int16_t>(def_level + 1), &child_reader));
- if (child_reader != nullptr) {
- children.push_back(std::move(child_reader));
- }
- }
-
- if (children.size() > 0) {
- *out = std::unique_ptr<ColumnReader::ColumnReaderImpl>(
- new StructImpl(children, def_level, pool_, node));
- }
- } else {
- // This should be a flat field case - translate the field index to
- // the correct column index by walking down to the leaf node
- const Node* walker = node;
- while (!walker->is_primitive()) {
- DCHECK(walker->is_group());
- auto group = static_cast<const GroupNode*>(walker);
- if (group->field_count() != 1) {
- return Status::NotImplemented("lists with structs are not supported.");
- }
- walker = group->field(0).get();
- }
- auto column_index = reader_->metadata()->schema()->ColumnIndex(*walker);
-
- // If the index of the column is found then a reader for the coliumn is needed.
- // Otherwise *out keeps the nullptr value.
- if (std::find(indices.begin(), indices.end(), column_index) != indices.end()) {
- std::unique_ptr<ColumnReader> reader;
- RETURN_NOT_OK(GetColumn(column_index, &reader));
- *out = std::move(reader->impl_);
- }
- }
-
- return Status::OK();
-}
-
-Status FileReader::Impl::ReadSchemaField(int i, std::shared_ptr<Array>* out) {
- std::vector<int> indices(reader_->metadata()->num_columns());
-
- for (size_t j = 0; j < indices.size(); ++j) {
- indices[j] = static_cast<int>(j);
- }
-
- return ReadSchemaField(i, indices, out);
-}
-
-Status FileReader::Impl::ReadSchemaField(int i, const std::vector<int>& indices,
- std::shared_ptr<Array>* out) {
- auto parquet_schema = reader_->metadata()->schema();
-
- auto node = parquet_schema->group_node()->field(i).get();
- std::unique_ptr<ColumnReader::ColumnReaderImpl> reader_impl;
-
- RETURN_NOT_OK(GetReaderForNode(i, node, indices, 1, &reader_impl));
- if (reader_impl == nullptr) {
- *out = nullptr;
- return Status::OK();
- }
-
- std::unique_ptr<ColumnReader> reader(new ColumnReader(std::move(reader_impl)));
-
- // TODO(wesm): This calculation doesn't make much sense when we have repeated
- // schema nodes
- int64_t records_to_read = 0;
-
- const FileMetaData& metadata = *reader_->metadata();
- for (int j = 0; j < metadata.num_row_groups(); j++) {
- records_to_read += metadata.RowGroup(j)->ColumnChunk(i)->num_values();
- }
-
- return reader->NextBatch(records_to_read, out);
-}
-
-Status FileReader::Impl::ReadColumn(int i, std::shared_ptr<Array>* out) {
- std::unique_ptr<ColumnReader> flat_column_reader;
- RETURN_NOT_OK(GetColumn(i, &flat_column_reader));
-
- int64_t records_to_read = 0;
- for (int j = 0; j < reader_->metadata()->num_row_groups(); j++) {
- records_to_read += reader_->metadata()->RowGroup(j)->ColumnChunk(i)->num_values();
- }
-
- return flat_column_reader->NextBatch(records_to_read, out);
-}
-
-Status FileReader::Impl::GetSchema(const std::vector<int>& indices,
- std::shared_ptr<::arrow::Schema>* out) {
- auto descr = reader_->metadata()->schema();
- auto parquet_key_value_metadata = reader_->metadata()->key_value_metadata();
- return FromParquetSchema(descr, indices, parquet_key_value_metadata, out);
-}
-
-Status FileReader::Impl::ReadColumnChunk(int column_index, int row_group_index,
- std::shared_ptr<Array>* out) {
- auto rg_metadata = reader_->metadata()->RowGroup(row_group_index);
- int64_t records_to_read = rg_metadata->ColumnChunk(column_index)->num_values();
-
- std::unique_ptr<FileColumnIterator> input(
- new SingleRowGroupIterator(column_index, row_group_index, reader_.get()));
-
- std::unique_ptr<ColumnReader::ColumnReaderImpl> impl(
- new PrimitiveImpl(pool_, std::move(input)));
- ColumnReader flat_column_reader(std::move(impl));
-
- std::shared_ptr<Array> array;
- RETURN_NOT_OK(flat_column_reader.NextBatch(records_to_read, &array));
- *out = array;
- return Status::OK();
-}
-
-Status FileReader::Impl::ReadRowGroup(int row_group_index,
- const std::vector<int>& indices,
- std::shared_ptr<Table>* out) {
- std::shared_ptr<::arrow::Schema> schema;
- RETURN_NOT_OK(GetSchema(indices, &schema));
-
- auto rg_metadata = reader_->metadata()->RowGroup(row_group_index);
-
- int num_columns = static_cast<int>(indices.size());
- std::vector<std::shared_ptr<Column>> columns(num_columns);
-
- // TODO(wesm): Refactor to share more code with ReadTable
-
- auto ReadColumnFunc = [&indices, &row_group_index, &schema, &columns, this](int i) {
- int column_index = indices[i];
-
- std::shared_ptr<Array> array;
- RETURN_NOT_OK(ReadColumnChunk(column_index, row_group_index, &array));
- columns[i] = std::make_shared<Column>(schema->field(i), array);
- return Status::OK();
- };
-
- if (use_threads_) {
- std::vector<std::future<Status>> futures;
- auto pool = ::arrow::internal::GetCpuThreadPool();
- for (int i = 0; i < num_columns; i++) {
- futures.push_back(pool->Submit(ReadColumnFunc, i));
- }
- Status final_status = Status::OK();
- for (auto& fut : futures) {
- Status st = fut.get();
- if (!st.ok()) {
- final_status = std::move(st);
- }
- }
- RETURN_NOT_OK(final_status);
- } else {
- for (int i = 0; i < num_columns; i++) {
- RETURN_NOT_OK(ReadColumnFunc(i));
- }
- }
-
- *out = Table::Make(schema, columns);
- return Status::OK();
-}
-
-Status FileReader::Impl::ReadTable(const std::vector<int>& indices,
- std::shared_ptr<Table>* out) {
- std::shared_ptr<::arrow::Schema> schema;
- RETURN_NOT_OK(GetSchema(indices, &schema));
-
- // We only need to read schema fields which have columns indicated
- // in the indices vector
- std::vector<int> field_indices;
- if (!ColumnIndicesToFieldIndices(*reader_->metadata()->schema(), indices,
- &field_indices)) {
- return Status::Invalid("Invalid column index");
- }
-
- int num_fields = static_cast<int>(field_indices.size());
- std::vector<std::shared_ptr<Column>> columns(num_fields);
-
- auto ReadColumnFunc = [&indices, &field_indices, &schema, &columns, this](int i) {
- std::shared_ptr<Array> array;
- RETURN_NOT_OK(ReadSchemaField(field_indices[i], indices, &array));
- columns[i] = std::make_shared<Column>(schema->field(i), array);
- return Status::OK();
- };
-
- if (use_threads_) {
- std::vector<std::future<Status>> futures;
- auto pool = ::arrow::internal::GetCpuThreadPool();
- for (int i = 0; i < num_fields; i++) {
- futures.push_back(pool->Submit(ReadColumnFunc, i));
- }
- Status final_status = Status::OK();
- for (auto& fut : futures) {
- Status st = fut.get();
- if (!st.ok()) {
- final_status = std::move(st);
- }
- }
- RETURN_NOT_OK(final_status);
- } else {
- for (int i = 0; i < num_fields; i++) {
- RETURN_NOT_OK(ReadColumnFunc(i));
- }
- }
-
- std::shared_ptr<Table> table = Table::Make(schema, columns);
- RETURN_NOT_OK(table->Validate());
- *out = table;
- return Status::OK();
-}
-
-Status FileReader::Impl::ReadTable(std::shared_ptr<Table>* table) {
- std::vector<int> indices(reader_->metadata()->num_columns());
-
- for (size_t i = 0; i < indices.size(); ++i) {
- indices[i] = static_cast<int>(i);
- }
- return ReadTable(indices, table);
-}
-
-Status FileReader::Impl::ReadRowGroups(const std::vector<int>& row_groups,
- const std::vector<int>& indices,
- std::shared_ptr<Table>* table) {
- // TODO(PARQUET-1393): Modify the record readers to already read this into a single,
- // continuous array.
- std::vector<std::shared_ptr<Table>> tables(row_groups.size(), nullptr);
-
- for (size_t i = 0; i < row_groups.size(); ++i) {
- RETURN_NOT_OK(ReadRowGroup(row_groups[i], indices, &tables[i]));
- }
- return ConcatenateTables(tables, table);
-}
-
-Status FileReader::Impl::ReadRowGroups(const std::vector<int>& row_groups,
- std::shared_ptr<Table>* table) {
- std::vector<int> indices(reader_->metadata()->num_columns());
-
- for (size_t i = 0; i < indices.size(); ++i) {
- indices[i] = static_cast<int>(i);
- }
- return ReadRowGroups(row_groups, indices, table);
-}
-
-Status FileReader::Impl::ReadRowGroup(int i, std::shared_ptr<Table>* table) {
- std::vector<int> indices(reader_->metadata()->num_columns());
-
- for (size_t i = 0; i < indices.size(); ++i) {
- indices[i] = static_cast<int>(i);
- }
- return ReadRowGroup(i, indices, table);
-}
-
-// Static ctor
-Status OpenFile(const std::shared_ptr<::arrow::io::ReadableFileInterface>& file,
- MemoryPool* allocator, const ReaderProperties& props,
- const std::shared_ptr<FileMetaData>& metadata,
- std::unique_ptr<FileReader>* reader) {
- std::unique_ptr<RandomAccessSource> io_wrapper(new ArrowInputFile(file));
- std::unique_ptr<ParquetReader> pq_reader;
- PARQUET_CATCH_NOT_OK(pq_reader =
- ParquetReader::Open(std::move(io_wrapper), props, metadata));
- reader->reset(new FileReader(allocator, std::move(pq_reader)));
- return Status::OK();
-}
-
-Status OpenFile(const std::shared_ptr<::arrow::io::ReadableFileInterface>& file,
- MemoryPool* allocator, std::unique_ptr<FileReader>* reader) {
- return OpenFile(file, allocator, ::parquet::default_reader_properties(), nullptr,
- reader);
-}
-
-Status FileReader::GetColumn(int i, std::unique_ptr<ColumnReader>* out) {
- return impl_->GetColumn(i, out);
-}
-
-Status FileReader::GetSchema(const std::vector<int>& indices,
- std::shared_ptr<::arrow::Schema>* out) {
- return impl_->GetSchema(indices, out);
-}
-
-Status FileReader::ReadColumn(int i, std::shared_ptr<Array>* out) {
- try {
- return impl_->ReadColumn(i, out);
- } catch (const ::parquet::ParquetException& e) {
- return ::arrow::Status::IOError(e.what());
- }
-}
-
-Status FileReader::ReadSchemaField(int i, std::shared_ptr<Array>* out) {
- try {
- return impl_->ReadSchemaField(i, out);
- } catch (const ::parquet::ParquetException& e) {
- return ::arrow::Status::IOError(e.what());
- }
-}
-
-Status FileReader::GetRecordBatchReader(const std::vector<int>& row_group_indices,
- std::shared_ptr<RecordBatchReader>* out) {
- std::vector<int> indices(impl_->num_columns());
-
- for (size_t j = 0; j < indices.size(); ++j) {
- indices[j] = static_cast<int>(j);
- }
-
- return GetRecordBatchReader(row_group_indices, indices, out);
-}
-
-Status FileReader::GetRecordBatchReader(const std::vector<int>& row_group_indices,
- const std::vector<int>& column_indices,
- std::shared_ptr<RecordBatchReader>* out) {
- // column indicies check
- std::shared_ptr<::arrow::Schema> schema;
- RETURN_NOT_OK(GetSchema(column_indices, &schema));
-
- // row group indices check
- int max_num = num_row_groups();
- for (auto row_group_index : row_group_indices) {
- if (row_group_index < 0 || row_group_index >= max_num) {
- std::ostringstream ss;
- ss << "Some index in row_group_indices is " << row_group_index
- << ", which is either < 0 or >= num_row_groups(" << max_num << ")";
- return Status::Invalid(ss.str());
- }
- }
-
- *out = std::make_shared<RowGroupRecordBatchReader>(row_group_indices, column_indices,
- schema, this);
- return Status::OK();
-}
-
-Status FileReader::ReadTable(std::shared_ptr<Table>* out) {
- try {
- return impl_->ReadTable(out);
- } catch (const ::parquet::ParquetException& e) {
- return ::arrow::Status::IOError(e.what());
- }
-}
-
-Status FileReader::ReadTable(const std::vector<int>& indices,
- std::shared_ptr<Table>* out) {
- try {
- return impl_->ReadTable(indices, out);
- } catch (const ::parquet::ParquetException& e) {
- return ::arrow::Status::IOError(e.what());
- }
-}
-
-Status FileReader::ReadRowGroup(int i, std::shared_ptr<Table>* out) {
- try {
- return impl_->ReadRowGroup(i, out);
- } catch (const ::parquet::ParquetException& e) {
- return ::arrow::Status::IOError(e.what());
- }
-}
-
-Status FileReader::ReadRowGroup(int i, const std::vector<int>& indices,
- std::shared_ptr<Table>* out) {
- try {
- return impl_->ReadRowGroup(i, indices, out);
- } catch (const ::parquet::ParquetException& e) {
- return ::arrow::Status::IOError(e.what());
- }
-}
-
-Status FileReader::ReadRowGroups(const std::vector<int>& row_groups,
- std::shared_ptr<Table>* out) {
- try {
- return impl_->ReadRowGroups(row_groups, out);
- } catch (const ::parquet::ParquetException& e) {
- return ::arrow::Status::IOError(e.what());
- }
-}
-
-Status FileReader::ReadRowGroups(const std::vector<int>& row_groups,
- const std::vector<int>& indices,
- std::shared_ptr<Table>* out) {
- try {
- return impl_->ReadRowGroups(row_groups, indices, out);
- } catch (const ::parquet::ParquetException& e) {
- return ::arrow::Status::IOError(e.what());
- }
-}
-
-std::shared_ptr<RowGroupReader> FileReader::RowGroup(int row_group_index) {
- return std::shared_ptr<RowGroupReader>(
- new RowGroupReader(impl_.get(), row_group_index));
-}
-
-int FileReader::num_row_groups() const { return impl_->num_row_groups(); }
-
-void FileReader::set_num_threads(int num_threads) {}
-
-void FileReader::set_use_threads(bool use_threads) {
- impl_->set_use_threads(use_threads);
-}
-
-Status FileReader::ScanContents(std::vector<int> columns, const int32_t column_batch_size,
- int64_t* num_rows) {
- try {
- *num_rows = ScanFileContents(columns, column_batch_size, impl_->reader());
- return Status::OK();
- } catch (const ::parquet::ParquetException& e) {
- return Status::IOError(e.what());
- }
-}
-
-const ParquetFileReader* FileReader::parquet_reader() const {
- return impl_->parquet_reader();
-}
-
-template <typename ParquetType>
-Status PrimitiveImpl::WrapIntoListArray(std::shared_ptr<Array>* array) {
- const int16_t* def_levels = record_reader_->def_levels();
- const int16_t* rep_levels = record_reader_->rep_levels();
- const int64_t total_levels_read = record_reader_->levels_position();
-
- std::shared_ptr<::arrow::Schema> arrow_schema;
- RETURN_NOT_OK(FromParquetSchema(input_->schema(), {input_->column_index()},
- input_->metadata()->key_value_metadata(),
- &arrow_schema));
- std::shared_ptr<Field> current_field = arrow_schema->field(0);
-
- if (descr_->max_repetition_level() > 0) {
- // Walk downwards to extract nullability
- std::vector<bool> nullable;
- std::vector<std::shared_ptr<::arrow::Int32Builder>> offset_builders;
- std::vector<std::shared_ptr<::arrow::BooleanBuilder>> valid_bits_builders;
- nullable.push_back(current_field->nullable());
- while (current_field->type()->num_children() > 0) {
- if (current_field->type()->num_children() > 1) {
- return Status::NotImplemented(
- "Fields with more than one child are not supported.");
- } else {
- if (current_field->type()->id() != ::arrow::Type::LIST) {
- return Status::NotImplemented(
- "Currently only nesting with Lists is supported.");
- }
- current_field = current_field->type()->child(0);
- }
- offset_builders.emplace_back(
- std::make_shared<::arrow::Int32Builder>(::arrow::int32(), pool_));
- valid_bits_builders.emplace_back(
- std::make_shared<::arrow::BooleanBuilder>(::arrow::boolean(), pool_));
- nullable.push_back(current_field->nullable());
- }
-
- int64_t list_depth = offset_builders.size();
- // This describes the minimal definition that describes a level that
- // reflects a value in the primitive values array.
- int16_t values_def_level = descr_->max_definition_level();
- if (nullable[nullable.size() - 1]) {
- values_def_level--;
- }
-
- // The definition levels that are needed so that a list is declared
- // as empty and not null.
- std::vector<int16_t> empty_def_level(list_depth);
- int def_level = 0;
- for (int i = 0; i < list_depth; i++) {
- if (nullable[i]) {
- def_level++;
- }
- empty_def_level[i] = static_cast<int16_t>(def_level);
- def_level++;
- }
-
- int32_t values_offset = 0;
- std::vector<int64_t> null_counts(list_depth, 0);
- for (int64_t i = 0; i < total_levels_read; i++) {
- int16_t rep_level = rep_levels[i];
- if (rep_level < descr_->max_repetition_level()) {
- for (int64_t j = rep_level; j < list_depth; j++) {
- if (j == (list_depth - 1)) {
- RETURN_NOT_OK(offset_builders[j]->Append(values_offset));
- } else {
- RETURN_NOT_OK(offset_builders[j]->Append(
- static_cast<int32_t>(offset_builders[j + 1]->length())));
- }
-
- if (((empty_def_level[j] - 1) == def_levels[i]) && (nullable[j])) {
- RETURN_NOT_OK(valid_bits_builders[j]->Append(false));
- null_counts[j]++;
- break;
- } else {
- RETURN_NOT_OK(valid_bits_builders[j]->Append(true));
- if (empty_def_level[j] == def_levels[i]) {
- break;
- }
- }
- }
- }
- if (def_levels[i] >= values_def_level) {
- values_offset++;
- }
- }
- // Add the final offset to all lists
- for (int64_t j = 0; j < list_depth; j++) {
- if (j == (list_depth - 1)) {
- RETURN_NOT_OK(offset_builders[j]->Append(values_offset));
- } else {
- RETURN_NOT_OK(offset_builders[j]->Append(
- static_cast<int32_t>(offset_builders[j + 1]->length())));
- }
- }
-
- std::vector<std::shared_ptr<Buffer>> offsets;
- std::vector<std::shared_ptr<Buffer>> valid_bits;
- std::vector<int64_t> list_lengths;
- for (int64_t j = 0; j < list_depth; j++) {
- list_lengths.push_back(offset_builders[j]->length() - 1);
- std::shared_ptr<Array> array;
- RETURN_NOT_OK(offset_builders[j]->Finish(&array));
- offsets.emplace_back(std::static_pointer_cast<Int32Array>(array)->values());
- RETURN_NOT_OK(valid_bits_builders[j]->Finish(&array));
- valid_bits.emplace_back(std::static_pointer_cast<BooleanArray>(array)->values());
- }
-
- std::shared_ptr<Array> output(*array);
- for (int64_t j = list_depth - 1; j >= 0; j--) {
- auto list_type =
- ::arrow::list(::arrow::field("item", output->type(), nullable[j + 1]));
- output = std::make_shared<::arrow::ListArray>(
- list_type, list_lengths[j], offsets[j], output, valid_bits[j], null_counts[j]);
- }
- *array = output;
- }
- return Status::OK();
-}
-
-template <typename ArrowType, typename ParquetType>
-struct supports_fast_path_impl {
- using ArrowCType = typename ArrowType::c_type;
- using ParquetCType = typename ParquetType::c_type;
- static constexpr bool value = std::is_same<ArrowCType, ParquetCType>::value;
-};
-
-template <typename ArrowType>
-struct supports_fast_path_impl<ArrowType, ByteArrayType> {
- static constexpr bool value = false;
-};
-
-template <typename ArrowType>
-struct supports_fast_path_impl<ArrowType, FLBAType> {
- static constexpr bool value = false;
-};
-
-template <typename ArrowType, typename ParquetType>
-using supports_fast_path =
- typename std::enable_if<supports_fast_path_impl<ArrowType, ParquetType>::value>::type;
-
-template <typename ArrowType, typename ParquetType, typename Enable = void>
-struct TransferFunctor {
- using ArrowCType = typename ArrowType::c_type;
- using ParquetCType = typename ParquetType::c_type;
-
- Status operator()(RecordReader* reader, MemoryPool* pool,
- const std::shared_ptr<::arrow::DataType>& type,
- std::shared_ptr<Array>* out) {
- static_assert(!std::is_same<ArrowType, ::arrow::Int32Type>::value,
- "The fast path transfer functor should be used "
- "for primitive values");
-
- int64_t length = reader->values_written();
- std::shared_ptr<Buffer> data;
- RETURN_NOT_OK(::arrow::AllocateBuffer(pool, length * sizeof(ArrowCType), &data));
-
- auto values = reinterpret_cast<const ParquetCType*>(reader->values());
- auto out_ptr = reinterpret_cast<ArrowCType*>(data->mutable_data());
- std::copy(values, values + length, out_ptr);
-
- if (reader->nullable_values()) {
- std::shared_ptr<ResizableBuffer> is_valid = reader->ReleaseIsValid();
- *out = std::make_shared<ArrayType<ArrowType>>(type, length, data, is_valid,
- reader->null_count());
- } else {
- *out = std::make_shared<ArrayType<ArrowType>>(type, length, data);
- }
- return Status::OK();
- }
-};
-
-template <typename ArrowType, typename ParquetType>
-struct TransferFunctor<ArrowType, ParquetType,
- supports_fast_path<ArrowType, ParquetType>> {
- Status operator()(RecordReader* reader, MemoryPool* pool,
- const std::shared_ptr<::arrow::DataType>& type,
- std::shared_ptr<Array>* out) {
- int64_t length = reader->values_written();
- std::shared_ptr<ResizableBuffer> values = reader->ReleaseValues();
-
- if (reader->nullable_values()) {
- std::shared_ptr<ResizableBuffer> is_valid = reader->ReleaseIsValid();
- *out = std::make_shared<ArrayType<ArrowType>>(type, length, values, is_valid,
- reader->null_count());
- } else {
- *out = std::make_shared<ArrayType<ArrowType>>(type, length, values);
- }
- return Status::OK();
- }
-};
-
-template <>
-struct TransferFunctor<::arrow::BooleanType, BooleanType> {
- Status operator()(RecordReader* reader, MemoryPool* pool,
- const std::shared_ptr<::arrow::DataType>& type,
- std::shared_ptr<Array>* out) {
- int64_t length = reader->values_written();
- std::shared_ptr<Buffer> data;
-
- const int64_t buffer_size = BytesForBits(length);
- RETURN_NOT_OK(::arrow::AllocateBuffer(pool, buffer_size, &data));
-
- // Transfer boolean values to packed bitmap
- auto values = reinterpret_cast<const bool*>(reader->values());
- uint8_t* data_ptr = data->mutable_data();
- memset(data_ptr, 0, buffer_size);
-
- for (int64_t i = 0; i < length; i++) {
- if (values[i]) {
- ::arrow::BitUtil::SetBit(data_ptr, i);
- }
- }
-
- if (reader->nullable_values()) {
- std::shared_ptr<ResizableBuffer> is_valid = reader->ReleaseIsValid();
- RETURN_NOT_OK(is_valid->Resize(BytesForBits(length), false));
- *out = std::make_shared<BooleanArray>(type, length, data, is_valid,
- reader->null_count());
- } else {
- *out = std::make_shared<BooleanArray>(type, length, data);
- }
- return Status::OK();
- }
-};
-
-template <>
-struct TransferFunctor<::arrow::TimestampType, Int96Type> {
- Status operator()(RecordReader* reader, MemoryPool* pool,
- const std::shared_ptr<::arrow::DataType>& type,
- std::shared_ptr<Array>* out) {
- int64_t length = reader->values_written();
- auto values = reinterpret_cast<const Int96*>(reader->values());
-
- std::shared_ptr<Buffer> data;
- RETURN_NOT_OK(::arrow::AllocateBuffer(pool, length * sizeof(int64_t), &data));
-
- auto data_ptr = reinterpret_cast<int64_t*>(data->mutable_data());
- for (int64_t i = 0; i < length; i++) {
- *data_ptr++ = impala_timestamp_to_nanoseconds(values[i]);
- }
-
- if (reader->nullable_values()) {
- std::shared_ptr<ResizableBuffer> is_valid = reader->ReleaseIsValid();
- *out = std::make_shared<TimestampArray>(type, length, data, is_valid,
- reader->null_count());
- } else {
- *out = std::make_shared<TimestampArray>(type, length, data);
- }
-
- return Status::OK();
- }
-};
-
-template <>
-struct TransferFunctor<::arrow::Date64Type, Int32Type> {
- Status operator()(RecordReader* reader, MemoryPool* pool,
- const std::shared_ptr<::arrow::DataType>& type,
- std::shared_ptr<Array>* out) {
- int64_t length = reader->values_written();
- auto values = reinterpret_cast<const int32_t*>(reader->values());
-
- std::shared_ptr<Buffer> data;
- RETURN_NOT_OK(::arrow::AllocateBuffer(pool, length * sizeof(int64_t), &data));
- auto out_ptr = reinterpret_cast<int64_t*>(data->mutable_data());
-
- for (int64_t i = 0; i < length; i++) {
- *out_ptr++ = static_cast<int64_t>(values[i]) * kMillisecondsInADay;
- }
-
- if (reader->nullable_values()) {
- std::shared_ptr<ResizableBuffer> is_valid = reader->ReleaseIsValid();
- *out = std::make_shared<::arrow::Date64Array>(type, length, data, is_valid,
- reader->null_count());
- } else {
- *out = std::make_shared<::arrow::Date64Array>(type, length, data);
- }
- return Status::OK();
- }
-};
-
-template <typename ArrowType, typename ParquetType>
-struct TransferFunctor<
- ArrowType, ParquetType,
- typename std::enable_if<std::is_same<ParquetType, ByteArrayType>::value ||
- std::is_same<ParquetType, FLBAType>::value>::type> {
- Status operator()(RecordReader* reader, MemoryPool* pool,
- const std::shared_ptr<::arrow::DataType>& type,
- std::shared_ptr<Array>* out) {
- RETURN_NOT_OK(reader->builder()->Finish(out));
-
- if (type->id() == ::arrow::Type::STRING) {
- // Convert from BINARY type to STRING
- auto new_data = (*out)->data()->Copy();
- new_data->type = type;
- *out = ::arrow::MakeArray(new_data);
- }
- return Status::OK();
- }
-};
-
-static uint64_t BytesToInteger(const uint8_t* bytes, int32_t start, int32_t stop) {
- using ::arrow::BitUtil::FromBigEndian;
-
- const int32_t length = stop - start;
-
- DCHECK_GE(length, 0);
- DCHECK_LE(length, 8);
-
- switch (length) {
- case 0:
- return 0;
- case 1:
- return bytes[start];
- case 2:
- return FromBigEndian(*reinterpret_cast<const uint16_t*>(bytes + start));
- case 3: {
- const uint64_t first_two_bytes =
- FromBigEndian(*reinterpret_cast<const uint16_t*>(bytes + start));
- const uint64_t last_byte = bytes[stop - 1];
- return first_two_bytes << 8 | last_byte;
- }
- case 4:
- return FromBigEndian(*reinterpret_cast<const uint32_t*>(bytes + start));
- case 5: {
- const uint64_t first_four_bytes =
- FromBigEndian(*reinterpret_cast<const uint32_t*>(bytes + start));
- const uint64_t last_byte = bytes[stop - 1];
- return first_four_bytes << 8 | last_byte;
- }
- case 6: {
- const uint64_t first_four_bytes =
- FromBigEndian(*reinterpret_cast<const uint32_t*>(bytes + start));
- const uint64_t last_two_bytes =
- FromBigEndian(*reinterpret_cast<const uint16_t*>(bytes + start + 4));
- return first_four_bytes << 16 | last_two_bytes;
- }
- case 7: {
- const uint64_t first_four_bytes =
- FromBigEndian(*reinterpret_cast<const uint32_t*>(bytes + start));
- const uint64_t second_two_bytes =
- FromBigEndian(*reinterpret_cast<const uint16_t*>(bytes + start + 4));
- const uint64_t last_byte = bytes[stop - 1];
- return first_four_bytes << 24 | second_two_bytes << 8 | last_byte;
- }
- case 8:
- return FromBigEndian(*reinterpret_cast<const uint64_t*>(bytes + start));
- default: {
- DCHECK(false);
- return UINT64_MAX;
- }
- }
-}
-
-static constexpr int32_t kMinDecimalBytes = 1;
-static constexpr int32_t kMaxDecimalBytes = 16;
-
-/// \brief Convert a sequence of big-endian bytes to one int64_t (high bits) and one
-/// uint64_t (low bits).
-static void BytesToIntegerPair(const uint8_t* bytes,
- const int32_t total_number_of_bytes_used, int64_t* high,
- uint64_t* low) {
- DCHECK_GE(total_number_of_bytes_used, kMinDecimalBytes);
- DCHECK_LE(total_number_of_bytes_used, kMaxDecimalBytes);
-
- /// Bytes are coming in big-endian, so the first byte is the MSB and therefore holds the
- /// sign bit.
- const bool is_negative = static_cast<int8_t>(bytes[0]) < 0;
-
- /// Sign extend the low bits if necessary
- *low = UINT64_MAX * (is_negative && total_number_of_bytes_used < 8);
- *high = -1 * (is_negative && total_number_of_bytes_used < kMaxDecimalBytes);
-
- /// Stop byte of the high bytes
- const int32_t high_bits_offset = std::max(0, total_number_of_bytes_used - 8);
-
- /// Shift left enough bits to make room for the incoming int64_t
- *high <<= high_bits_offset * CHAR_BIT;
-
- /// Preserve the upper bits by inplace OR-ing the int64_t
- *high |= BytesToInteger(bytes, 0, high_bits_offset);
-
- /// Stop byte of the low bytes
- const int32_t low_bits_offset = std::min(total_number_of_bytes_used, 8);
-
- /// Shift left enough bits to make room for the incoming uint64_t
- *low <<= low_bits_offset * CHAR_BIT;
-
- /// Preserve the upper bits by inplace OR-ing the uint64_t
- *low |= BytesToInteger(bytes, high_bits_offset, total_number_of_bytes_used);
-}
-
-static inline void RawBytesToDecimalBytes(const uint8_t* value, int32_t byte_width,
- uint8_t* out_buf) {
- // view the first 8 bytes as an unsigned 64-bit integer
- auto low = reinterpret_cast<uint64_t*>(out_buf);
-
- // view the second 8 bytes as a signed 64-bit integer
- auto high = reinterpret_cast<int64_t*>(out_buf + sizeof(uint64_t));
-
- // Convert the fixed size binary array bytes into a Decimal128 compatible layout
- BytesToIntegerPair(value, byte_width, high, low);
-}
-
-/// \brief Convert an array of FixedLenByteArrays to an arrow::Decimal128Array
-/// We do this by:
-/// 1. Creating a arrow::FixedSizeBinaryArray from the RecordReader's builder
-/// 2. Allocating a buffer for the arrow::Decimal128Array
-/// 3. Converting the big-endian bytes in the FixedSizeBinaryArray to two integers
-/// representing the high and low bits of each decimal value.
-template <>
-struct TransferFunctor<::arrow::Decimal128Type, FLBAType> {
- Status operator()(RecordReader* reader, MemoryPool* pool,
- const std::shared_ptr<::arrow::DataType>& type,
- std::shared_ptr<Array>* out) {
- DCHECK_EQ(type->id(), ::arrow::Type::DECIMAL);
-
- // Finish the built data into a temporary array
- std::shared_ptr<Array> array;
- RETURN_NOT_OK(reader->builder()->Finish(&array));
- const auto& fixed_size_binary_array =
- static_cast<const ::arrow::FixedSizeBinaryArray&>(*array);
-
- // Get the byte width of the values in the FixedSizeBinaryArray. Most of the time
- // this will be different from the decimal array width because we write the minimum
- // number of bytes necessary to represent a given precision
- const int32_t byte_width =
- static_cast<const ::arrow::FixedSizeBinaryType&>(*fixed_size_binary_array.type())
- .byte_width();
-
- // The byte width of each decimal value
- const int32_t type_length =
- static_cast<const ::arrow::Decimal128Type&>(*type).byte_width();
-
- // number of elements in the entire array
- const int64_t length = fixed_size_binary_array.length();
-
- // allocate memory for the decimal array
- std::shared_ptr<Buffer> data;
- RETURN_NOT_OK(::arrow::AllocateBuffer(pool, length * type_length, &data));
-
- // raw bytes that we can write to
- uint8_t* out_ptr = data->mutable_data();
-
- // convert each FixedSizeBinary value to valid decimal bytes
- const int64_t null_count = fixed_size_binary_array.null_count();
- if (null_count > 0) {
- for (int64_t i = 0; i < length; ++i, out_ptr += type_length) {
- if (!fixed_size_binary_array.IsNull(i)) {
- RawBytesToDecimalBytes(fixed_size_binary_array.GetValue(i), byte_width,
- out_ptr);
- }
- }
- } else {
- for (int64_t i = 0; i < length; ++i, out_ptr += type_length) {
- RawBytesToDecimalBytes(fixed_size_binary_array.GetValue(i), byte_width, out_ptr);
- }
- }
-
- *out = std::make_shared<::arrow::Decimal128Array>(
- type, length, data, fixed_size_binary_array.null_bitmap(), null_count);
- return Status::OK();
- }
-};
-
-/// \brief Convert an Int32 or Int64 array into a Decimal128Array
-/// The parquet spec allows systems to write decimals in int32, int64 if the values are
-/// small enough to fit in less 4 bytes or less than 8 bytes, respectively.
-/// This function implements the conversion from int32 and int64 arrays to decimal arrays.
-template <typename ParquetIntegerType,
- typename = typename std::enable_if<
- std::is_same<ParquetIntegerType, Int32Type>::value ||
- std::is_same<ParquetIntegerType, Int64Type>::value>::type>
-static Status DecimalIntegerTransfer(RecordReader* reader, MemoryPool* pool,
- const std::shared_ptr<::arrow::DataType>& type,
- std::shared_ptr<Array>* out) {
- DCHECK_EQ(type->id(), ::arrow::Type::DECIMAL);
-
- const int64_t length = reader->values_written();
-
- using ElementType = typename ParquetIntegerType::c_type;
- static_assert(std::is_same<ElementType, int32_t>::value ||
- std::is_same<ElementType, int64_t>::value,
- "ElementType must be int32_t or int64_t");
-
- const auto values = reinterpret_cast<const ElementType*>(reader->values());
-
- const auto& decimal_type = static_cast<const ::arrow::Decimal128Type&>(*type);
- const int64_t type_length = decimal_type.byte_width();
-
- std::shared_ptr<Buffer> data;
- RETURN_NOT_OK(::arrow::AllocateBuffer(pool, length * type_length, &data));
- uint8_t* out_ptr = data->mutable_data();
-
- using ::arrow::BitUtil::FromLittleEndian;
-
- for (int64_t i = 0; i < length; ++i, out_ptr += type_length) {
- // sign/zero extend int32_t values, otherwise a no-op
- const auto value = static_cast<int64_t>(values[i]);
-
- auto out_ptr_view = reinterpret_cast<uint64_t*>(out_ptr);
-
- // No-op on little endian machines, byteswap on big endian
- out_ptr_view[0] = FromLittleEndian(static_cast<uint64_t>(value));
-
- // no need to byteswap here because we're sign/zero extending exactly 8 bytes
- out_ptr_view[1] = static_cast<uint64_t>(value < 0 ? -1 : 0);
- }
-
- if (reader->nullable_values()) {
- std::shared_ptr<ResizableBuffer> is_valid = reader->ReleaseIsValid();
- *out = std::make_shared<::arrow::Decimal128Array>(type, length, data, is_valid,
- reader->null_count());
- } else {
- *out = std::make_shared<::arrow::Decimal128Array>(type, length, data);
- }
- return Status::OK();
-}
-
-template <>
-struct TransferFunctor<::arrow::Decimal128Type, Int32Type> {
- Status operator()(RecordReader* reader, MemoryPool* pool,
- const std::shared_ptr<::arrow::DataType>& type,
- std::shared_ptr<Array>* out) {
- return DecimalIntegerTransfer<Int32Type>(reader, pool, type, out);
- }
-};
-
-template <>
-struct TransferFunctor<::arrow::Decimal128Type, Int64Type> {
- Status operator()(RecordReader* reader, MemoryPool* pool,
- const std::shared_ptr<::arrow::DataType>& type,
- std::shared_ptr<Array>* out) {
- return DecimalIntegerTransfer<Int64Type>(reader, pool, type, out);
- }
-};
-
-#define TRANSFER_DATA(ArrowType, ParquetType) \
- TransferFunctor<ArrowType, ParquetType> func; \
- RETURN_NOT_OK(func(record_reader_.get(), pool_, field_->type(), out)); \
- RETURN_NOT_OK(WrapIntoListArray<ParquetType>(out))
-
-#define TRANSFER_CASE(ENUM, ArrowType, ParquetType) \
- case ::arrow::Type::ENUM: { \
- TRANSFER_DATA(ArrowType, ParquetType); \
- } break;
-
-Status PrimitiveImpl::NextBatch(int64_t records_to_read, std::shared_ptr<Array>* out) {
- try {
- // Pre-allocation gives much better performance for flat columns
- record_reader_->Reserve(records_to_read);
-
- record_reader_->Reset();
- while (records_to_read > 0) {
- if (!record_reader_->HasMoreData()) {
- break;
- }
- int64_t records_read = record_reader_->ReadRecords(records_to_read);
- records_to_read -= records_read;
- if (records_read == 0) {
- NextRowGroup();
- }
- }
- } catch (const ::parquet::ParquetException& e) {
- return ::arrow::Status::IOError(e.what());
- }
-
- switch (field_->type()->id()) {
- TRANSFER_CASE(BOOL, ::arrow::BooleanType, BooleanType)
- TRANSFER_CASE(UINT8, ::arrow::UInt8Type, Int32Type)
- TRANSFER_CASE(INT8, ::arrow::Int8Type, Int32Type)
- TRANSFER_CASE(UINT16, ::arrow::UInt16Type, Int32Type)
- TRANSFER_CASE(INT16, ::arrow::Int16Type, Int32Type)
- TRANSFER_CASE(UINT32, ::arrow::UInt32Type, Int32Type)
- TRANSFER_CASE(INT32, ::arrow::Int32Type, Int32Type)
- TRANSFER_CASE(UINT64, ::arrow::UInt64Type, Int64Type)
- TRANSFER_CASE(INT64, ::arrow::Int64Type, Int64Type)
- TRANSFER_CASE(FLOAT, ::arrow::FloatType, FloatType)
- TRANSFER_CASE(DOUBLE, ::arrow::DoubleType, DoubleType)
- TRANSFER_CASE(STRING, ::arrow::StringType, ByteArrayType)
- TRANSFER_CASE(BINARY, ::arrow::BinaryType, ByteArrayType)
- TRANSFER_CASE(DATE32, ::arrow::Date32Type, Int32Type)
- TRANSFER_CASE(DATE64, ::arrow::Date64Type, Int32Type)
- TRANSFER_CASE(FIXED_SIZE_BINARY, ::arrow::FixedSizeBinaryType, FLBAType)
- case ::arrow::Type::NA: {
- *out = std::make_shared<::arrow::NullArray>(record_reader_->values_written());
- RETURN_NOT_OK(WrapIntoListArray<Int32Type>(out));
- break;
- }
- case ::arrow::Type::DECIMAL: {
- switch (descr_->physical_type()) {
- case ::parquet::Type::INT32: {
- TRANSFER_DATA(::arrow::Decimal128Type, Int32Type);
- } break;
- case ::parquet::Type::INT64: {
- TRANSFER_DATA(::arrow::Decimal128Type, Int64Type);
- } break;
- case ::parquet::Type::FIXED_LEN_BYTE_ARRAY: {
- TRANSFER_DATA(::arrow::Decimal128Type, FLBAType);
- } break;
- default:
- return Status::Invalid(
- "Physical type for decimal must be int32, int64, or fixed length binary");
- }
- } break;
- case ::arrow::Type::TIMESTAMP: {
- ::arrow::TimestampType* timestamp_type =
- static_cast<::arrow::TimestampType*>(field_->type().get());
- switch (timestamp_type->unit()) {
- case ::arrow::TimeUnit::MILLI:
- case ::arrow::TimeUnit::MICRO: {
- TRANSFER_DATA(::arrow::TimestampType, Int64Type);
- } break;
- case ::arrow::TimeUnit::NANO: {
- TRANSFER_DATA(::arrow::TimestampType, Int96Type);
- } break;
- default:
- return Status::NotImplemented("TimeUnit not supported");
- }
- } break;
- TRANSFER_CASE(TIME32, ::arrow::Time32Type, Int32Type)
- TRANSFER_CASE(TIME64, ::arrow::Time64Type, Int64Type)
- default:
- std::stringstream ss;
- ss << "No support for reading columns of type " << field_->type()->ToString();
- return Status::NotImplemented(ss.str());
- }
-
- return Status::OK();
-}
-
-void PrimitiveImpl::NextRowGroup() {
- std::unique_ptr<PageReader> page_reader = input_->NextChunk();
- record_reader_->SetPageReader(std::move(page_reader));
-}
-
-Status PrimitiveImpl::GetDefLevels(const int16_t** data, size_t* length) {
- *data = record_reader_->def_levels();
- *length = record_reader_->levels_written();
- return Status::OK();
-}
-
-Status PrimitiveImpl::GetRepLevels(const int16_t** data, size_t* length) {
- *data = record_reader_->rep_levels();
- *length = record_reader_->levels_written();
- return Status::OK();
-}
-
-ColumnReader::ColumnReader(std::unique_ptr<ColumnReaderImpl> impl)
- : impl_(std::move(impl)) {}
-
-ColumnReader::~ColumnReader() {}
-
-Status ColumnReader::NextBatch(int64_t records_to_read, std::shared_ptr<Array>* out) {
- return impl_->NextBatch(records_to_read, out);
-}
-
-// StructImpl methods
-
-Status StructImpl::DefLevelsToNullArray(std::shared_ptr<Buffer>* null_bitmap_out,
- int64_t* null_count_out) {
- std::shared_ptr<Buffer> null_bitmap;
- auto null_count = 0;
- const int16_t* def_levels_data;
- size_t def_levels_length;
- RETURN_NOT_OK(GetDefLevels(&def_levels_data, &def_levels_length));
- RETURN_NOT_OK(AllocateEmptyBitmap(pool_, def_levels_length, &null_bitmap));
- uint8_t* null_bitmap_ptr = null_bitmap->mutable_data();
- for (size_t i = 0; i < def_levels_length; i++) {
- if (def_levels_data[i] < struct_def_level_) {
- // Mark null
- null_count += 1;
- } else {
- DCHECK_EQ(def_levels_data[i], struct_def_level_);
- ::arrow::BitUtil::SetBit(null_bitmap_ptr, i);
- }
- }
-
- *null_count_out = null_count;
- *null_bitmap_out = (null_count == 0) ? nullptr : null_bitmap;
- return Status::OK();
-}
-
-// TODO(itaiin): Consider caching the results of this calculation -
-// note that this is only used once for each read for now
-Status StructImpl::GetDefLevels(const int16_t** data, size_t* length) {
- *data = nullptr;
- if (children_.size() == 0) {
- // Empty struct
- *length = 0;
- return Status::OK();
- }
-
- // We have at least one child
- const int16_t* child_def_levels;
- size_t child_length;
- RETURN_NOT_OK(children_[0]->GetDefLevels(&child_def_levels, &child_length));
- auto size = child_length * sizeof(int16_t);
- RETURN_NOT_OK(AllocateResizableBuffer(pool_, size, &def_levels_buffer_));
- // Initialize with the minimal def level
- std::memset(def_levels_buffer_->mutable_data(), -1, size);
- auto result_levels = reinterpret_cast<int16_t*>(def_levels_buffer_->mutable_data());
-
- // When a struct is defined, all of its children def levels are at least at
- // nesting level, and def level equals nesting level.
- // When a struct is not defined, all of its children def levels are less than
- // the nesting level, and the def level equals max(children def levels)
- // All other possibilities are malformed definition data.
- for (auto& child : children_) {
- size_t current_child_length;
- RETURN_NOT_OK(child->GetDefLevels(&child_def_levels, ¤t_child_length));
- DCHECK_EQ(child_length, current_child_length);
- for (size_t i = 0; i < child_length; i++) {
- // Check that value is either uninitialized, or current
- // and previous children def levels agree on the struct level
- DCHECK((result_levels[i] == -1) || ((result_levels[i] >= struct_def_level_) ==
- (child_def_levels[i] >= struct_def_level_)));
- result_levels[i] =
- std::max(result_levels[i], std::min(child_def_levels[i], struct_def_level_));
- }
- }
- *data = reinterpret_cast<const int16_t*>(def_levels_buffer_->data());
- *length = child_length;
- return Status::OK();
-}
-
-void StructImpl::InitField(
- const Node* node, const std::vector<std::shared_ptr<ColumnReaderImpl>>& children) {
- // Make a shallow node to field conversion from the children fields
- std::vector<std::shared_ptr<::arrow::Field>> fields(children.size());
- for (size_t i = 0; i < children.size(); i++) {
- fields[i] = children[i]->field();
- }
- auto type = ::arrow::struct_(fields);
- field_ = ::arrow::field(node->name(), type);
-}
-
-Status StructImpl::GetRepLevels(const int16_t** data, size_t* length) {
- return Status::NotImplemented("GetRepLevels is not implemented for struct");
-}
-
-Status StructImpl::NextBatch(int64_t records_to_read, std::shared_ptr<Array>* out) {
- std::vector<std::shared_ptr<Array>> children_arrays;
- std::shared_ptr<Buffer> null_bitmap;
- int64_t null_count;
-
- // Gather children arrays and def levels
- for (auto& child : children_) {
- std::shared_ptr<Array> child_array;
-
- RETURN_NOT_OK(child->NextBatch(records_to_read, &child_array));
- children_arrays.push_back(child_array);
- }
-
- RETURN_NOT_OK(DefLevelsToNullArray(&null_bitmap, &null_count));
-
- int64_t struct_length = children_arrays[0]->length();
- for (size_t i = 1; i < children_arrays.size(); ++i) {
- if (children_arrays[i]->length() != struct_length) {
- // TODO(wesm): This should really only occur if the Parquet file is
- // malformed. Should this be a DCHECK?
- return Status::Invalid("Struct children had different lengths");
- }
- }
-
- *out = std::make_shared<StructArray>(field()->type(), struct_length, children_arrays,
- null_bitmap, null_count);
- return Status::OK();
-}
-
-std::shared_ptr<ColumnChunkReader> RowGroupReader::Column(int column_index) {
- return std::shared_ptr<ColumnChunkReader>(
- new ColumnChunkReader(impl_, row_group_index_, column_index));
-}
-
-Status RowGroupReader::ReadTable(const std::vector<int>& column_indices,
- std::shared_ptr<::arrow::Table>* out) {
- return impl_->ReadRowGroup(row_group_index_, column_indices, out);
-}
-
-Status RowGroupReader::ReadTable(std::shared_ptr<::arrow::Table>* out) {
- return impl_->ReadRowGroup(row_group_index_, out);
-}
-
-RowGroupReader::~RowGroupReader() {}
-
-RowGroupReader::RowGroupReader(FileReader::Impl* impl, int row_group_index)
- : impl_(impl), row_group_index_(row_group_index) {}
-
-Status ColumnChunkReader::Read(std::shared_ptr<::arrow::Array>* out) {
- return impl_->ReadColumnChunk(column_index_, row_group_index_, out);
-}
-
-ColumnChunkReader::~ColumnChunkReader() {}
-
-ColumnChunkReader::ColumnChunkReader(FileReader::Impl* impl, int row_group_index,
- int column_index)
- : impl_(impl), column_index_(column_index), row_group_index_(row_group_index) {}
-
-} // namespace arrow
-} // namespace parquet
diff --git a/src/parquet/arrow/reader.h b/src/parquet/arrow/reader.h
deleted file mode 100644
index db135da..0000000
--- a/src/parquet/arrow/reader.h
+++ /dev/null
@@ -1,306 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#ifndef PARQUET_ARROW_READER_H
-#define PARQUET_ARROW_READER_H
-
-#include <memory>
-#include <vector>
-
-#include "parquet/api/reader.h"
-#include "parquet/api/schema.h"
-
-#include "arrow/io/interfaces.h"
-
-namespace arrow {
-
-class Array;
-class MemoryPool;
-class RecordBatchReader;
-class Status;
-class Table;
-} // namespace arrow
-
-namespace parquet {
-
-namespace arrow {
-
-class ColumnChunkReader;
-class ColumnReader;
-class RowGroupReader;
-
-// Arrow read adapter class for deserializing Parquet files as Arrow row
-// batches.
-//
-// This interfaces caters for different use cases and thus provides different
-// interfaces. In its most simplistic form, we cater for a user that wants to
-// read the whole Parquet at once with the FileReader::ReadTable method.
-//
-// More advanced users that also want to implement parallelism on top of each
-// single Parquet files should do this on the RowGroup level. For this, they can
-// call FileReader::RowGroup(i)->ReadTable to receive only the specified
-// RowGroup as a table.
-//
-// In the most advanced situation, where a consumer wants to independently read
-// RowGroups in parallel and consume each column individually, they can call
-// FileReader::RowGroup(i)->Column(j)->Read and receive an arrow::Column
-// instance.
-//
-// TODO(wesm): nested data does not always make sense with this user
-// interface unless you are only reading a single leaf node from a branch of
-// a table. For example:
-//
-// repeated group data {
-// optional group record {
-// optional int32 val1;
-// optional byte_array val2;
-// optional bool val3;
-// }
-// optional int32 val4;
-// }
-//
-// In the Parquet file, there are 3 leaf nodes:
-//
-// * data.record.val1
-// * data.record.val2
-// * data.record.val3
-// * data.val4
-//
-// When materializing this data in an Arrow array, we would have:
-//
-// data: list<struct<
-// record: struct<
-// val1: int32,
-// val2: string (= list<uint8>),
-// val3: bool,
-// >,
-// val4: int32
-// >>
-//
-// However, in the Parquet format, each leaf node has its own repetition and
-// definition levels describing the structure of the intermediate nodes in
-// this array structure. Thus, we will need to scan the leaf data for a group
-// of leaf nodes part of the same type tree to create a single result Arrow
-// nested array structure.
-//
-// This is additionally complicated "chunky" repeated fields or very large byte
-// arrays
-class PARQUET_EXPORT FileReader {
- public:
- FileReader(::arrow::MemoryPool* pool, std::unique_ptr<ParquetFileReader> reader);
-
- // Since the distribution of columns amongst a Parquet file's row groups may
- // be uneven (the number of values in each column chunk can be different), we
- // provide a column-oriented read interface. The ColumnReader hides the
- // details of paging through the file's row groups and yielding
- // fully-materialized arrow::Array instances
- //
- // Returns error status if the column of interest is not flat.
- ::arrow::Status GetColumn(int i, std::unique_ptr<ColumnReader>* out);
-
- /// \brief Return arrow schema by apply selection of column indices.
- /// \returns error status if passed wrong indices.
- ::arrow::Status GetSchema(const std::vector<int>& indices,
- std::shared_ptr<::arrow::Schema>* out);
-
- // Read column as a whole into an Array.
- ::arrow::Status ReadColumn(int i, std::shared_ptr<::arrow::Array>* out);
-
- // NOTE: Experimental API
- // Reads a specific top level schema field into an Array
- // The index i refers the index of the top level schema field, which may
- // be nested or flat - e.g.
- //
- // 0 foo.bar
- // foo.bar.baz
- // foo.qux
- // 1 foo2
- // 2 foo3
- //
- // i=0 will read the entire foo struct, i=1 the foo2 primitive column etc
- ::arrow::Status ReadSchemaField(int i, std::shared_ptr<::arrow::Array>* out);
-
- // NOTE: Experimental API
- // Reads a specific top level schema field into an Array, while keeping only chosen
- // leaf columns.
- // The index i refers the index of the top level schema field, which may
- // be nested or flat, and indices vector refers to the leaf column indices - e.g.
- //
- // i indices
- // 0 0 foo.bar
- // 0 1 foo.bar.baz
- // 0 2 foo.qux
- // 1 3 foo2
- // 2 4 foo3
- //
- // i=0 indices={0,2} will read a partial struct with foo.bar and foo.quox columns
- // i=1 indices={3} will read foo2 column
- // i=1 indices={2} will result in out=nullptr
- // leaf indices which are unrelated to the schema field are ignored
- ::arrow::Status ReadSchemaField(int i, const std::vector<int>& indices,
- std::shared_ptr<::arrow::Array>* out);
-
- /// \brief Return a RecordBatchReader of row groups selected from row_group_indices, the
- /// ordering in row_group_indices matters.
- /// \returns error Status if row_group_indices contains invalid index
- ::arrow::Status GetRecordBatchReader(const std::vector<int>& row_group_indices,
- std::shared_ptr<::arrow::RecordBatchReader>* out);
-
- /// \brief Return a RecordBatchReader of row groups selected from row_group_indices,
- /// whose columns are selected by column_indices. The ordering in row_group_indices
- /// and column_indices matter.
- /// \returns error Status if either row_group_indices or column_indices contains invalid
- /// index
- ::arrow::Status GetRecordBatchReader(const std::vector<int>& row_group_indices,
- const std::vector<int>& column_indices,
- std::shared_ptr<::arrow::RecordBatchReader>* out);
-
- // Read a table of columns into a Table
- ::arrow::Status ReadTable(std::shared_ptr<::arrow::Table>* out);
-
- // Read a table of columns into a Table. Read only the indicated column
- // indices (relative to the schema)
- ::arrow::Status ReadTable(const std::vector<int>& column_indices,
- std::shared_ptr<::arrow::Table>* out);
-
- ::arrow::Status ReadRowGroup(int i, const std::vector<int>& column_indices,
- std::shared_ptr<::arrow::Table>* out);
-
- ::arrow::Status ReadRowGroup(int i, std::shared_ptr<::arrow::Table>* out);
-
- ::arrow::Status ReadRowGroups(const std::vector<int>& row_groups,
- const std::vector<int>& column_indices,
- std::shared_ptr<::arrow::Table>* out);
-
- ::arrow::Status ReadRowGroups(const std::vector<int>& row_groups,
- std::shared_ptr<::arrow::Table>* out);
-
- /// \brief Scan file contents with one thread, return number of rows
- ::arrow::Status ScanContents(std::vector<int> columns, const int32_t column_batch_size,
- int64_t* num_rows);
-
- /// \brief Return a reader for the RowGroup, this object must not outlive the
- /// FileReader.
- std::shared_ptr<RowGroupReader> RowGroup(int row_group_index);
-
- int num_row_groups() const;
-
- const ParquetFileReader* parquet_reader() const;
-
- /// Set the number of threads to use during reads of multiple columns. By
- /// default only 1 thread is used
- /// \deprecated Use set_use_threads instead.
- void set_num_threads(int num_threads);
-
- /// Set whether to use multiple threads during reads of multiple columns.
- /// By default only one thread is used.
- void set_use_threads(bool use_threads);
-
- virtual ~FileReader();
-
- private:
- friend ColumnChunkReader;
- friend RowGroupReader;
-
- class PARQUET_NO_EXPORT Impl;
- std::unique_ptr<Impl> impl_;
-};
-
-class PARQUET_EXPORT RowGroupReader {
- public:
- std::shared_ptr<ColumnChunkReader> Column(int column_index);
-
- ::arrow::Status ReadTable(const std::vector<int>& column_indices,
- std::shared_ptr<::arrow::Table>* out);
- ::arrow::Status ReadTable(std::shared_ptr<::arrow::Table>* out);
-
- virtual ~RowGroupReader();
-
- private:
- friend FileReader;
- RowGroupReader(FileReader::Impl* reader, int row_group_index);
-
- FileReader::Impl* impl_;
- int row_group_index_;
-};
-
-class PARQUET_EXPORT ColumnChunkReader {
- public:
- ::arrow::Status Read(std::shared_ptr<::arrow::Array>* out);
-
- virtual ~ColumnChunkReader();
-
- private:
- friend RowGroupReader;
- ColumnChunkReader(FileReader::Impl* impl, int row_group_index, int column_index);
-
- FileReader::Impl* impl_;
- int column_index_;
- int row_group_index_;
-};
-
-// At this point, the column reader is a stream iterator. It only knows how to
-// read the next batch of values for a particular column from the file until it
-// runs out.
-//
-// We also do not expose any internal Parquet details, such as row groups. This
-// might change in the future.
-class PARQUET_EXPORT ColumnReader {
- public:
- class PARQUET_NO_EXPORT ColumnReaderImpl;
- virtual ~ColumnReader();
-
- // Scan the next array of the indicated size. The actual size of the
- // returned array may be less than the passed size depending how much data is
- // available in the file.
- //
- // When all the data in the file has been exhausted, the result is set to
- // nullptr.
- //
- // Returns Status::OK on a successful read, including if you have exhausted
- // the data available in the file.
- ::arrow::Status NextBatch(int64_t batch_size, std::shared_ptr<::arrow::Array>* out);
-
- private:
- std::unique_ptr<ColumnReaderImpl> impl_;
- explicit ColumnReader(std::unique_ptr<ColumnReaderImpl> impl);
-
- friend class FileReader;
- friend class PrimitiveImpl;
- friend class StructImpl;
-};
-
-// Helper function to create a file reader from an implementation of an Arrow
-// readable file
-//
-// metadata : separately-computed file metadata, can be nullptr
-PARQUET_EXPORT
-::arrow::Status OpenFile(const std::shared_ptr<::arrow::io::ReadableFileInterface>& file,
- ::arrow::MemoryPool* allocator,
- const ReaderProperties& properties,
- const std::shared_ptr<FileMetaData>& metadata,
- std::unique_ptr<FileReader>* reader);
-
-PARQUET_EXPORT
-::arrow::Status OpenFile(const std::shared_ptr<::arrow::io::ReadableFileInterface>& file,
- ::arrow::MemoryPool* allocator,
- std::unique_ptr<FileReader>* reader);
-
-} // namespace arrow
-} // namespace parquet
-
-#endif // PARQUET_ARROW_READER_H
diff --git a/src/parquet/arrow/record_reader.cc b/src/parquet/arrow/record_reader.cc
deleted file mode 100644
index 781e1ba..0000000
--- a/src/parquet/arrow/record_reader.cc
+++ /dev/null
@@ -1,817 +0,0 @@
-// licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#include "parquet/arrow/record_reader.h"
-
-#include <algorithm>
-#include <cstdint>
-#include <memory>
-#include <sstream>
-
-#include <arrow/buffer.h>
-#include <arrow/memory_pool.h>
-#include <arrow/status.h>
-#include <arrow/util/bit-util.h>
-#include <arrow/util/rle-encoding.h>
-
-#include "parquet/column_page.h"
-#include "parquet/column_reader.h"
-#include "parquet/encoding-internal.h"
-#include "parquet/exception.h"
-#include "parquet/properties.h"
-
-using arrow::MemoryPool;
-
-namespace parquet {
-namespace internal {
-
-namespace BitUtil = ::arrow::BitUtil;
-
-template <typename DType>
-class TypedRecordReader;
-
-// PLAIN_DICTIONARY is deprecated but used to be used as a dictionary index
-// encoding.
-static bool IsDictionaryIndexEncoding(const Encoding::type& e) {
- return e == Encoding::RLE_DICTIONARY || e == Encoding::PLAIN_DICTIONARY;
-}
-
-class RecordReader::RecordReaderImpl {
- public:
- RecordReaderImpl(const ColumnDescriptor* descr, MemoryPool* pool)
- : descr_(descr),
- pool_(pool),
- num_buffered_values_(0),
- num_decoded_values_(0),
- max_def_level_(descr->max_definition_level()),
- max_rep_level_(descr->max_repetition_level()),
- at_record_start_(true),
- records_read_(0),
- values_written_(0),
- values_capacity_(0),
- null_count_(0),
- levels_written_(0),
- levels_position_(0),
- levels_capacity_(0) {
- nullable_values_ = internal::HasSpacedValues(descr);
- values_ = AllocateBuffer(pool);
- valid_bits_ = AllocateBuffer(pool);
- def_levels_ = AllocateBuffer(pool);
- rep_levels_ = AllocateBuffer(pool);
-
- if (descr->physical_type() == Type::BYTE_ARRAY) {
- builder_.reset(new ::arrow::BinaryBuilder(pool));
- } else if (descr->physical_type() == Type::FIXED_LEN_BYTE_ARRAY) {
- int byte_width = descr->type_length();
- std::shared_ptr<::arrow::DataType> type = ::arrow::fixed_size_binary(byte_width);
- builder_.reset(new ::arrow::FixedSizeBinaryBuilder(type, pool));
- }
- Reset();
- }
-
- virtual ~RecordReaderImpl() = default;
-
- virtual int64_t ReadRecords(int64_t num_records) = 0;
-
- // Dictionary decoders must be reset when advancing row groups
- virtual void ResetDecoders() = 0;
-
- void SetPageReader(std::unique_ptr<PageReader> reader) {
- at_record_start_ = true;
- pager_ = std::move(reader);
- ResetDecoders();
- }
-
- bool HasMoreData() const { return pager_ != nullptr; }
-
- int16_t* def_levels() const {
- return reinterpret_cast<int16_t*>(def_levels_->mutable_data());
- }
-
- int16_t* rep_levels() {
- return reinterpret_cast<int16_t*>(rep_levels_->mutable_data());
- }
-
- uint8_t* values() const { return values_->mutable_data(); }
-
- /// \brief Number of values written including nulls (if any)
- int64_t values_written() const { return values_written_; }
-
- int64_t levels_position() const { return levels_position_; }
- int64_t levels_written() const { return levels_written_; }
-
- // We may outwardly have the appearance of having exhausted a column chunk
- // when in fact we are in the middle of processing the last batch
- bool has_values_to_process() const { return levels_position_ < levels_written_; }
-
- int64_t null_count() const { return null_count_; }
-
- bool nullable_values() const { return nullable_values_; }
-
- std::shared_ptr<ResizableBuffer> ReleaseValues() {
- auto result = values_;
- values_ = AllocateBuffer(pool_);
- return result;
- }
-
- std::shared_ptr<ResizableBuffer> ReleaseIsValid() {
- auto result = valid_bits_;
- valid_bits_ = AllocateBuffer(pool_);
- return result;
- }
-
- ::arrow::ArrayBuilder* builder() { return builder_.get(); }
-
- // Process written repetition/definition levels to reach the end of
- // records. Process no more levels than necessary to delimit the indicated
- // number of logical records. Updates internal state of RecordReader
- //
- // \return Number of records delimited
- int64_t DelimitRecords(int64_t num_records, int64_t* values_seen) {
- int64_t values_to_read = 0;
- int64_t records_read = 0;
-
- const int16_t* def_levels = this->def_levels() + levels_position_;
- const int16_t* rep_levels = this->rep_levels() + levels_position_;
-
- DCHECK_GT(max_rep_level_, 0);
-
- // Count logical records and number of values to read
- while (levels_position_ < levels_written_) {
- if (*rep_levels++ == 0) {
- // If at_record_start_ is true, we are seeing the start of a record
- // for the second time, such as after repeated calls to
- // DelimitRecords. In this case we must continue until we find
- // another record start or exhausting the ColumnChunk
- if (!at_record_start_) {
- // We've reached the end of a record; increment the record count.
- ++records_read;
- if (records_read == num_records) {
- // We've found the number of records we were looking for. Set
- // at_record_start_ to true and break
- at_record_start_ = true;
- break;
- }
- }
- }
-
- // We have decided to consume the level at this position; therefore we
- // must advance until we find another record boundary
- at_record_start_ = false;
-
- if (*def_levels++ == max_def_level_) {
- ++values_to_read;
- }
- ++levels_position_;
- }
- *values_seen = values_to_read;
- return records_read;
- }
-
- // Read multiple definition levels into preallocated memory
- //
- // Returns the number of decoded definition levels
- int64_t ReadDefinitionLevels(int64_t batch_size, int16_t* levels) {
- if (descr_->max_definition_level() == 0) {
- return 0;
- }
- return definition_level_decoder_.Decode(static_cast<int>(batch_size), levels);
- }
-
- int64_t ReadRepetitionLevels(int64_t batch_size, int16_t* levels) {
- if (descr_->max_repetition_level() == 0) {
- return 0;
- }
- return repetition_level_decoder_.Decode(static_cast<int>(batch_size), levels);
- }
-
- int64_t available_values_current_page() const {
- return num_buffered_values_ - num_decoded_values_;
- }
-
- void ConsumeBufferedValues(int64_t num_values) { num_decoded_values_ += num_values; }
-
- Type::type type() const { return descr_->physical_type(); }
-
- const ColumnDescriptor* descr() const { return descr_; }
-
- void Reserve(int64_t capacity) {
- ReserveLevels(capacity);
- ReserveValues(capacity);
- }
-
- void ReserveLevels(int64_t capacity) {
- if (descr_->max_definition_level() > 0 &&
- (levels_written_ + capacity > levels_capacity_)) {
- int64_t new_levels_capacity = BitUtil::NextPower2(levels_capacity_ + 1);
- while (levels_written_ + capacity > new_levels_capacity) {
- new_levels_capacity = BitUtil::NextPower2(new_levels_capacity + 1);
- }
- PARQUET_THROW_NOT_OK(
- def_levels_->Resize(new_levels_capacity * sizeof(int16_t), false));
- if (descr_->max_repetition_level() > 0) {
- PARQUET_THROW_NOT_OK(
- rep_levels_->Resize(new_levels_capacity * sizeof(int16_t), false));
- }
- levels_capacity_ = new_levels_capacity;
- }
- }
-
- void ReserveValues(int64_t capacity) {
- if (values_written_ + capacity > values_capacity_) {
- int64_t new_values_capacity = BitUtil::NextPower2(values_capacity_ + 1);
- while (values_written_ + capacity > new_values_capacity) {
- new_values_capacity = BitUtil::NextPower2(new_values_capacity + 1);
- }
-
- int type_size = GetTypeByteSize(descr_->physical_type());
- PARQUET_THROW_NOT_OK(values_->Resize(new_values_capacity * type_size, false));
- values_capacity_ = new_values_capacity;
- }
- if (nullable_values_) {
- int64_t valid_bytes_new = BitUtil::BytesForBits(values_capacity_);
- if (valid_bits_->size() < valid_bytes_new) {
- int64_t valid_bytes_old = BitUtil::BytesForBits(values_written_);
- PARQUET_THROW_NOT_OK(valid_bits_->Resize(valid_bytes_new, false));
-
- // Avoid valgrind warnings
- memset(valid_bits_->mutable_data() + valid_bytes_old, 0,
- valid_bytes_new - valid_bytes_old);
- }
- }
- }
-
- void Reset() {
- ResetValues();
-
- if (levels_written_ > 0) {
- const int64_t levels_remaining = levels_written_ - levels_position_;
- // Shift remaining levels to beginning of buffer and trim to only the number
- // of decoded levels remaining
- int16_t* def_data = def_levels();
- int16_t* rep_data = rep_levels();
-
- std::copy(def_data + levels_position_, def_data + levels_written_, def_data);
- std::copy(rep_data + levels_position_, rep_data + levels_written_, rep_data);
-
- PARQUET_THROW_NOT_OK(
- def_levels_->Resize(levels_remaining * sizeof(int16_t), false));
- PARQUET_THROW_NOT_OK(
- rep_levels_->Resize(levels_remaining * sizeof(int16_t), false));
-
- levels_written_ -= levels_position_;
- levels_position_ = 0;
- levels_capacity_ = levels_remaining;
- }
-
- records_read_ = 0;
-
- // Calling Finish on the builders also resets them
- }
-
- void ResetValues() {
- if (values_written_ > 0) {
- // Resize to 0, but do not shrink to fit
- PARQUET_THROW_NOT_OK(values_->Resize(0, false));
- PARQUET_THROW_NOT_OK(valid_bits_->Resize(0, false));
- values_written_ = 0;
- values_capacity_ = 0;
- null_count_ = 0;
- }
- }
-
- protected:
- const ColumnDescriptor* descr_;
- ::arrow::MemoryPool* pool_;
-
- std::unique_ptr<PageReader> pager_;
- std::shared_ptr<Page> current_page_;
-
- // Not set if full schema for this field has no optional or repeated elements
- LevelDecoder definition_level_decoder_;
-
- // Not set for flat schemas.
- LevelDecoder repetition_level_decoder_;
-
- // The total number of values stored in the data page. This is the maximum of
- // the number of encoded definition levels or encoded values. For
- // non-repeated, required columns, this is equal to the number of encoded
- // values. For repeated or optional values, there may be fewer data values
- // than levels, and this tells you how many encoded levels there are in that
- // case.
- int64_t num_buffered_values_;
-
- // The number of values from the current data page that have been decoded
- // into memory
- int64_t num_decoded_values_;
-
- const int16_t max_def_level_;
- const int16_t max_rep_level_;
-
- bool nullable_values_;
-
- bool at_record_start_;
- int64_t records_read_;
-
- int64_t values_written_;
- int64_t values_capacity_;
- int64_t null_count_;
-
- int64_t levels_written_;
- int64_t levels_position_;
- int64_t levels_capacity_;
-
- // TODO(wesm): ByteArray / FixedLenByteArray types
- std::unique_ptr<::arrow::ArrayBuilder> builder_;
-
- std::shared_ptr<::arrow::ResizableBuffer> values_;
-
- template <typename T>
- T* ValuesHead() {
- return reinterpret_cast<T*>(values_->mutable_data()) + values_written_;
- }
-
- std::shared_ptr<::arrow::ResizableBuffer> valid_bits_;
- std::shared_ptr<::arrow::ResizableBuffer> def_levels_;
- std::shared_ptr<::arrow::ResizableBuffer> rep_levels_;
-};
-
-// The minimum number of repetition/definition levels to decode at a time, for
-// better vectorized performance when doing many smaller record reads
-constexpr int64_t kMinLevelBatchSize = 1024;
-
-template <typename DType>
-class TypedRecordReader : public RecordReader::RecordReaderImpl {
- public:
- typedef typename DType::c_type T;
-
- TypedRecordReader(const ColumnDescriptor* schema, ::arrow::MemoryPool* pool)
- : RecordReader::RecordReaderImpl(schema, pool), current_decoder_(nullptr) {}
-
- void ResetDecoders() override { decoders_.clear(); }
-
- inline void ReadValuesSpaced(int64_t values_with_nulls, int64_t null_count) {
- uint8_t* valid_bits = valid_bits_->mutable_data();
- const int64_t valid_bits_offset = values_written_;
-
- int64_t num_decoded = current_decoder_->DecodeSpaced(
- ValuesHead<T>(), static_cast<int>(values_with_nulls),
- static_cast<int>(null_count), valid_bits, valid_bits_offset);
- DCHECK_EQ(num_decoded, values_with_nulls);
- }
-
- inline void ReadValuesDense(int64_t values_to_read) {
- int64_t num_decoded =
- current_decoder_->Decode(ValuesHead<T>(), static_cast<int>(values_to_read));
- DCHECK_EQ(num_decoded, values_to_read);
- }
-
- // Return number of logical records read
- int64_t ReadRecordData(const int64_t num_records) {
- // Conservative upper bound
- const int64_t possible_num_values =
- std::max(num_records, levels_written_ - levels_position_);
- ReserveValues(possible_num_values);
-
- const int64_t start_levels_position = levels_position_;
-
- int64_t values_to_read = 0;
- int64_t records_read = 0;
- if (max_rep_level_ > 0) {
- records_read = DelimitRecords(num_records, &values_to_read);
- } else if (max_def_level_ > 0) {
- // No repetition levels, skip delimiting logic. Each level represents a
- // null or not null entry
- records_read = std::min(levels_written_ - levels_position_, num_records);
-
- // This is advanced by DelimitRecords, which we skipped
- levels_position_ += records_read;
- } else {
- records_read = values_to_read = num_records;
- }
-
- int64_t null_count = 0;
- if (nullable_values_) {
- int64_t values_with_nulls = 0;
- internal::DefinitionLevelsToBitmap(
- def_levels() + start_levels_position, levels_position_ - start_levels_position,
- max_def_level_, max_rep_level_, &values_with_nulls, &null_count,
- valid_bits_->mutable_data(), values_written_);
- values_to_read = values_with_nulls - null_count;
- ReadValuesSpaced(values_with_nulls, null_count);
- ConsumeBufferedValues(levels_position_ - start_levels_position);
- } else {
- ReadValuesDense(values_to_read);
- ConsumeBufferedValues(values_to_read);
- }
- // Total values, including null spaces, if any
- values_written_ += values_to_read + null_count;
- null_count_ += null_count;
-
- return records_read;
- }
-
- // Returns true if there are still values in this column.
- bool HasNext() {
- // Either there is no data page available yet, or the data page has been
- // exhausted
- if (num_buffered_values_ == 0 || num_decoded_values_ == num_buffered_values_) {
- if (!ReadNewPage() || num_buffered_values_ == 0) {
- return false;
- }
- }
- return true;
- }
-
- int64_t ReadRecords(int64_t num_records) override {
- // Delimit records, then read values at the end
- int64_t records_read = 0;
-
- if (levels_position_ < levels_written_) {
- records_read += ReadRecordData(num_records);
- }
-
- int64_t level_batch_size = std::max(kMinLevelBatchSize, num_records);
-
- // If we are in the middle of a record, we continue until reaching the
- // desired number of records or the end of the current record if we've found
- // enough records
- while (!at_record_start_ || records_read < num_records) {
- // Is there more data to read in this row group?
- if (!HasNext()) {
- if (!at_record_start_) {
- // We ended the row group while inside a record that we haven't seen
- // the end of yet. So increment the record count for the last record in
- // the row group
- ++records_read;
- at_record_start_ = true;
- }
- break;
- }
-
- /// We perform multiple batch reads until we either exhaust the row group
- /// or observe the desired number of records
- int64_t batch_size = std::min(level_batch_size, available_values_current_page());
-
- // No more data in column
- if (batch_size == 0) {
- break;
- }
-
- if (max_def_level_ > 0) {
- ReserveLevels(batch_size);
-
- int16_t* def_levels = this->def_levels() + levels_written_;
- int16_t* rep_levels = this->rep_levels() + levels_written_;
-
- // Not present for non-repeated fields
- int64_t levels_read = 0;
- if (max_rep_level_ > 0) {
- levels_read = ReadDefinitionLevels(batch_size, def_levels);
- if (ReadRepetitionLevels(batch_size, rep_levels) != levels_read) {
- throw ParquetException("Number of decoded rep / def levels did not match");
- }
- } else if (max_def_level_ > 0) {
- levels_read = ReadDefinitionLevels(batch_size, def_levels);
- }
-
- // Exhausted column chunk
- if (levels_read == 0) {
- break;
- }
-
- levels_written_ += levels_read;
- records_read += ReadRecordData(num_records - records_read);
- } else {
- // No repetition or definition levels
- batch_size = std::min(num_records - records_read, batch_size);
- records_read += ReadRecordData(batch_size);
- }
- }
-
- return records_read;
- }
-
- private:
- typedef Decoder<DType> DecoderType;
-
- // Map of encoding type to the respective decoder object. For example, a
- // column chunk's data pages may include both dictionary-encoded and
- // plain-encoded data.
- std::unordered_map<int, std::shared_ptr<DecoderType>> decoders_;
-
- DecoderType* current_decoder_;
-
- // Advance to the next data page
- bool ReadNewPage();
-
- void ConfigureDictionary(const DictionaryPage* page);
-};
-
-template <>
-inline void TypedRecordReader<ByteArrayType>::ReadValuesDense(int64_t values_to_read) {
- auto values = ValuesHead<ByteArray>();
- int64_t num_decoded =
- current_decoder_->Decode(values, static_cast<int>(values_to_read));
- DCHECK_EQ(num_decoded, values_to_read);
-
- auto builder = static_cast<::arrow::BinaryBuilder*>(builder_.get());
- for (int64_t i = 0; i < num_decoded; i++) {
- PARQUET_THROW_NOT_OK(
- builder->Append(values[i].ptr, static_cast<int64_t>(values[i].len)));
- }
- ResetValues();
-}
-
-template <>
-inline void TypedRecordReader<FLBAType>::ReadValuesDense(int64_t values_to_read) {
- auto values = ValuesHead<FLBA>();
- int64_t num_decoded =
- current_decoder_->Decode(values, static_cast<int>(values_to_read));
- DCHECK_EQ(num_decoded, values_to_read);
-
- auto builder = static_cast<::arrow::FixedSizeBinaryBuilder*>(builder_.get());
- for (int64_t i = 0; i < num_decoded; i++) {
- PARQUET_THROW_NOT_OK(builder->Append(values[i].ptr));
- }
- ResetValues();
-}
-
-template <>
-inline void TypedRecordReader<ByteArrayType>::ReadValuesSpaced(int64_t values_to_read,
- int64_t null_count) {
- uint8_t* valid_bits = valid_bits_->mutable_data();
- const int64_t valid_bits_offset = values_written_;
- auto values = ValuesHead<ByteArray>();
-
- int64_t num_decoded = current_decoder_->DecodeSpaced(
- values, static_cast<int>(values_to_read), static_cast<int>(null_count), valid_bits,
- valid_bits_offset);
- DCHECK_EQ(num_decoded, values_to_read);
-
- auto builder = static_cast<::arrow::BinaryBuilder*>(builder_.get());
-
- for (int64_t i = 0; i < num_decoded; i++) {
- if (::arrow::BitUtil::GetBit(valid_bits, valid_bits_offset + i)) {
- PARQUET_THROW_NOT_OK(
- builder->Append(values[i].ptr, static_cast<int64_t>(values[i].len)));
- } else {
- PARQUET_THROW_NOT_OK(builder->AppendNull());
- }
- }
- ResetValues();
-}
-
-template <>
-inline void TypedRecordReader<FLBAType>::ReadValuesSpaced(int64_t values_to_read,
- int64_t null_count) {
- uint8_t* valid_bits = valid_bits_->mutable_data();
- const int64_t valid_bits_offset = values_written_;
- auto values = ValuesHead<FLBA>();
-
- int64_t num_decoded = current_decoder_->DecodeSpaced(
- values, static_cast<int>(values_to_read), static_cast<int>(null_count), valid_bits,
- valid_bits_offset);
- DCHECK_EQ(num_decoded, values_to_read);
-
- auto builder = static_cast<::arrow::FixedSizeBinaryBuilder*>(builder_.get());
- for (int64_t i = 0; i < num_decoded; i++) {
- if (::arrow::BitUtil::GetBit(valid_bits, valid_bits_offset + i)) {
- PARQUET_THROW_NOT_OK(builder->Append(values[i].ptr));
- } else {
- PARQUET_THROW_NOT_OK(builder->AppendNull());
- }
- }
- ResetValues();
-}
-
-template <typename DType>
-inline void TypedRecordReader<DType>::ConfigureDictionary(const DictionaryPage* page) {
- int encoding = static_cast<int>(page->encoding());
- if (page->encoding() == Encoding::PLAIN_DICTIONARY ||
- page->encoding() == Encoding::PLAIN) {
- encoding = static_cast<int>(Encoding::RLE_DICTIONARY);
- }
-
- auto it = decoders_.find(encoding);
- if (it != decoders_.end()) {
- throw ParquetException("Column cannot have more than one dictionary.");
- }
-
- if (page->encoding() == Encoding::PLAIN_DICTIONARY ||
- page->encoding() == Encoding::PLAIN) {
- PlainDecoder<DType> dictionary(descr_);
- dictionary.SetData(page->num_values(), page->data(), page->size());
-
- // The dictionary is fully decoded during DictionaryDecoder::Init, so the
- // DictionaryPage buffer is no longer required after this step
- //
- // TODO(wesm): investigate whether this all-or-nothing decoding of the
- // dictionary makes sense and whether performance can be improved
-
- auto decoder = std::make_shared<DictionaryDecoder<DType>>(descr_, pool_);
- decoder->SetDict(&dictionary);
- decoders_[encoding] = decoder;
- } else {
- ParquetException::NYI("only plain dictionary encoding has been implemented");
- }
-
- current_decoder_ = decoders_[encoding].get();
-}
-
-template <typename DType>
-bool TypedRecordReader<DType>::ReadNewPage() {
- // Loop until we find the next data page.
- const uint8_t* buffer;
-
- while (true) {
- current_page_ = pager_->NextPage();
- if (!current_page_) {
- // EOS
- return false;
- }
-
- if (current_page_->type() == PageType::DICTIONARY_PAGE) {
- ConfigureDictionary(static_cast<const DictionaryPage*>(current_page_.get()));
- continue;
- } else if (current_page_->type() == PageType::DATA_PAGE) {
- const DataPage* page = static_cast<const DataPage*>(current_page_.get());
-
- // Read a data page.
- num_buffered_values_ = page->num_values();
-
- // Have not decoded any values from the data page yet
- num_decoded_values_ = 0;
-
- buffer = page->data();
-
- // If the data page includes repetition and definition levels, we
- // initialize the level decoder and subtract the encoded level bytes from
- // the page size to determine the number of bytes in the encoded data.
- int64_t data_size = page->size();
-
- // Data page Layout: Repetition Levels - Definition Levels - encoded values.
- // Levels are encoded as rle or bit-packed.
- // Init repetition levels
- if (descr_->max_repetition_level() > 0) {
- int64_t rep_levels_bytes = repetition_level_decoder_.SetData(
- page->repetition_level_encoding(), descr_->max_repetition_level(),
- static_cast<int>(num_buffered_values_), buffer);
- buffer += rep_levels_bytes;
- data_size -= rep_levels_bytes;
- }
- // TODO figure a way to set max_definition_level_ to 0
- // if the initial value is invalid
-
- // Init definition levels
- if (descr_->max_definition_level() > 0) {
- int64_t def_levels_bytes = definition_level_decoder_.SetData(
- page->definition_level_encoding(), descr_->max_definition_level(),
- static_cast<int>(num_buffered_values_), buffer);
- buffer += def_levels_bytes;
- data_size -= def_levels_bytes;
- }
-
- // Get a decoder object for this page or create a new decoder if this is the
- // first page with this encoding.
- Encoding::type encoding = page->encoding();
-
- if (IsDictionaryIndexEncoding(encoding)) {
- encoding = Encoding::RLE_DICTIONARY;
- }
-
- auto it = decoders_.find(static_cast<int>(encoding));
- if (it != decoders_.end()) {
- if (encoding == Encoding::RLE_DICTIONARY) {
- DCHECK(current_decoder_->encoding() == Encoding::RLE_DICTIONARY);
- }
- current_decoder_ = it->second.get();
- } else {
- switch (encoding) {
- case Encoding::PLAIN: {
- std::shared_ptr<DecoderType> decoder(new PlainDecoder<DType>(descr_));
- decoders_[static_cast<int>(encoding)] = decoder;
- current_decoder_ = decoder.get();
- break;
- }
- case Encoding::RLE_DICTIONARY:
- throw ParquetException("Dictionary page must be before data page.");
-
- case Encoding::DELTA_BINARY_PACKED:
- case Encoding::DELTA_LENGTH_BYTE_ARRAY:
- case Encoding::DELTA_BYTE_ARRAY:
- ParquetException::NYI("Unsupported encoding");
-
- default:
- throw ParquetException("Unknown encoding type.");
- }
- }
- current_decoder_->SetData(static_cast<int>(num_buffered_values_), buffer,
- static_cast<int>(data_size));
- return true;
- } else {
- // We don't know what this page type is. We're allowed to skip non-data
- // pages.
- continue;
- }
- }
- return true;
-}
-
-std::shared_ptr<RecordReader> RecordReader::Make(const ColumnDescriptor* descr,
- MemoryPool* pool) {
- switch (descr->physical_type()) {
- case Type::BOOLEAN:
- return std::shared_ptr<RecordReader>(
- new RecordReader(new TypedRecordReader<BooleanType>(descr, pool)));
- case Type::INT32:
- return std::shared_ptr<RecordReader>(
- new RecordReader(new TypedRecordReader<Int32Type>(descr, pool)));
- case Type::INT64:
- return std::shared_ptr<RecordReader>(
- new RecordReader(new TypedRecordReader<Int64Type>(descr, pool)));
- case Type::INT96:
- return std::shared_ptr<RecordReader>(
- new RecordReader(new TypedRecordReader<Int96Type>(descr, pool)));
- case Type::FLOAT:
- return std::shared_ptr<RecordReader>(
- new RecordReader(new TypedRecordReader<FloatType>(descr, pool)));
- case Type::DOUBLE:
- return std::shared_ptr<RecordReader>(
- new RecordReader(new TypedRecordReader<DoubleType>(descr, pool)));
- case Type::BYTE_ARRAY:
- return std::shared_ptr<RecordReader>(
- new RecordReader(new TypedRecordReader<ByteArrayType>(descr, pool)));
- case Type::FIXED_LEN_BYTE_ARRAY:
- return std::shared_ptr<RecordReader>(
- new RecordReader(new TypedRecordReader<FLBAType>(descr, pool)));
- default:
- DCHECK(false);
- }
- // Unreachable code, but supress compiler warning
- return nullptr;
-}
-
-// ----------------------------------------------------------------------
-// Implement public API
-
-RecordReader::RecordReader(RecordReaderImpl* impl) { impl_.reset(impl); }
-
-RecordReader::~RecordReader() {}
-
-int64_t RecordReader::ReadRecords(int64_t num_records) {
- return impl_->ReadRecords(num_records);
-}
-
-void RecordReader::Reset() { return impl_->Reset(); }
-
-void RecordReader::Reserve(int64_t num_values) { impl_->Reserve(num_values); }
-
-const int16_t* RecordReader::def_levels() const { return impl_->def_levels(); }
-
-const int16_t* RecordReader::rep_levels() const { return impl_->rep_levels(); }
-
-const uint8_t* RecordReader::values() const { return impl_->values(); }
-
-std::shared_ptr<ResizableBuffer> RecordReader::ReleaseValues() {
- return impl_->ReleaseValues();
-}
-
-std::shared_ptr<ResizableBuffer> RecordReader::ReleaseIsValid() {
- return impl_->ReleaseIsValid();
-}
-
-::arrow::ArrayBuilder* RecordReader::builder() { return impl_->builder(); }
-
-int64_t RecordReader::values_written() const { return impl_->values_written(); }
-
-int64_t RecordReader::levels_position() const { return impl_->levels_position(); }
-
-int64_t RecordReader::levels_written() const { return impl_->levels_written(); }
-
-int64_t RecordReader::null_count() const { return impl_->null_count(); }
-
-bool RecordReader::nullable_values() const { return impl_->nullable_values(); }
-
-bool RecordReader::HasMoreData() const { return impl_->HasMoreData(); }
-
-void RecordReader::SetPageReader(std::unique_ptr<PageReader> reader) {
- impl_->SetPageReader(std::move(reader));
-}
-
-} // namespace internal
-} // namespace parquet
diff --git a/src/parquet/arrow/record_reader.h b/src/parquet/arrow/record_reader.h
deleted file mode 100644
index f02bf05..0000000
--- a/src/parquet/arrow/record_reader.h
+++ /dev/null
@@ -1,113 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#ifndef PARQUET_RECORD_READER_H
-#define PARQUET_RECORD_READER_H
-
-#include <cstdint>
-#include <cstring>
-#include <iostream>
-#include <memory>
-#include <unordered_map>
-#include <vector>
-
-#include <arrow/buffer.h>
-#include <arrow/builder.h>
-#include <arrow/memory_pool.h>
-#include <arrow/util/bit-util.h>
-
-#include "parquet/column_reader.h"
-#include "parquet/schema.h"
-#include "parquet/util/visibility.h"
-
-namespace parquet {
-namespace internal {
-
-/// \brief Stateful column reader that delimits semantic records for both flat
-/// and nested columns
-///
-/// \note API EXPERIMENTAL
-/// \since 1.3.0
-class RecordReader {
- public:
- // So that we can create subclasses
- class RecordReaderImpl;
-
- static std::shared_ptr<RecordReader> Make(
- const ColumnDescriptor* descr,
- ::arrow::MemoryPool* pool = ::arrow::default_memory_pool());
-
- virtual ~RecordReader();
-
- /// \brief Decoded definition levels
- const int16_t* def_levels() const;
-
- /// \brief Decoded repetition levels
- const int16_t* rep_levels() const;
-
- /// \brief Decoded values, including nulls, if any
- const uint8_t* values() const;
-
- /// \brief Attempt to read indicated number of records from column chunk
- /// \return number of records read
- int64_t ReadRecords(int64_t num_records);
-
- /// \brief Pre-allocate space for data. Results in better flat read performance
- void Reserve(int64_t num_values);
-
- /// \brief Clear consumed values and repetition/definition levels as the
- /// result of calling ReadRecords
- void Reset();
-
- std::shared_ptr<ResizableBuffer> ReleaseValues();
- std::shared_ptr<ResizableBuffer> ReleaseIsValid();
- ::arrow::ArrayBuilder* builder();
-
- /// \brief Number of values written including nulls (if any)
- int64_t values_written() const;
-
- /// \brief Number of definition / repetition levels (from those that have
- /// been decoded) that have been consumed inside the reader.
- int64_t levels_position() const;
-
- /// \brief Number of definition / repetition levels that have been written
- /// internally in the reader
- int64_t levels_written() const;
-
- /// \brief Number of nulls in the leaf
- int64_t null_count() const;
-
- /// \brief True if the leaf values are nullable
- bool nullable_values() const;
-
- /// \brief Return true if the record reader has more internal data yet to
- /// process
- bool HasMoreData() const;
-
- /// \brief Advance record reader to the next row group
- /// \param[in] reader obtained from RowGroupReader::GetColumnPageReader
- void SetPageReader(std::unique_ptr<PageReader> reader);
-
- private:
- std::unique_ptr<RecordReaderImpl> impl_;
- explicit RecordReader(RecordReaderImpl* impl);
-};
-
-} // namespace internal
-} // namespace parquet
-
-#endif // PARQUET_RECORD_READER_H
diff --git a/src/parquet/arrow/schema.cc b/src/parquet/arrow/schema.cc
deleted file mode 100644
index 48b1181..0000000
--- a/src/parquet/arrow/schema.cc
+++ /dev/null
@@ -1,704 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#include "parquet/arrow/schema.h"
-
-#include <string>
-#include <unordered_set>
-#include <vector>
-
-#include "parquet/api/schema.h"
-#include "parquet/util/schema-util.h"
-
-#include "arrow/api.h"
-#include "arrow/util/logging.h"
-
-using arrow::Field;
-using arrow::Status;
-using arrow::TypePtr;
-
-using ArrowType = arrow::Type;
-
-using parquet::Repetition;
-using parquet::schema::GroupNode;
-using parquet::schema::Node;
-using parquet::schema::NodePtr;
-using parquet::schema::PrimitiveNode;
-
-using ParquetType = parquet::Type;
-using parquet::LogicalType;
-
-namespace parquet {
-
-namespace arrow {
-
-const auto TIMESTAMP_MS = ::arrow::timestamp(::arrow::TimeUnit::MILLI);
-const auto TIMESTAMP_US = ::arrow::timestamp(::arrow::TimeUnit::MICRO);
-const auto TIMESTAMP_NS = ::arrow::timestamp(::arrow::TimeUnit::NANO);
-
-TypePtr MakeDecimal128Type(const PrimitiveNode& node) {
- const auto& metadata = node.decimal_metadata();
- return ::arrow::decimal(metadata.precision, metadata.scale);
-}
-
-static Status FromByteArray(const PrimitiveNode& node, TypePtr* out) {
- switch (node.logical_type()) {
- case LogicalType::UTF8:
- *out = ::arrow::utf8();
- break;
- case LogicalType::DECIMAL:
- *out = MakeDecimal128Type(node);
- break;
- default:
- // BINARY
- *out = ::arrow::binary();
- break;
- }
- return Status::OK();
-}
-
-static Status FromFLBA(const PrimitiveNode& node, TypePtr* out) {
- switch (node.logical_type()) {
- case LogicalType::NONE:
- *out = ::arrow::fixed_size_binary(node.type_length());
- break;
- case LogicalType::DECIMAL:
- *out = MakeDecimal128Type(node);
- break;
- default:
- std::stringstream ss;
- ss << "Unhandled logical type " << LogicalTypeToString(node.logical_type())
- << " for fixed-length binary array";
- return Status::NotImplemented(ss.str());
- }
-
- return Status::OK();
-}
-
-static Status FromInt32(const PrimitiveNode& node, TypePtr* out) {
- switch (node.logical_type()) {
- case LogicalType::NONE:
- *out = ::arrow::int32();
- break;
- case LogicalType::UINT_8:
- *out = ::arrow::uint8();
- break;
- case LogicalType::INT_8:
- *out = ::arrow::int8();
- break;
- case LogicalType::UINT_16:
- *out = ::arrow::uint16();
- break;
- case LogicalType::INT_16:
- *out = ::arrow::int16();
- break;
- case LogicalType::INT_32:
- *out = ::arrow::int32();
- break;
- case LogicalType::UINT_32:
- *out = ::arrow::uint32();
- break;
- case LogicalType::DATE:
- *out = ::arrow::date32();
- break;
- case LogicalType::TIME_MILLIS:
- *out = ::arrow::time32(::arrow::TimeUnit::MILLI);
- break;
- case LogicalType::DECIMAL:
- *out = MakeDecimal128Type(node);
- break;
- default:
- std::stringstream ss;
- ss << "Unhandled logical type " << LogicalTypeToString(node.logical_type())
- << " for INT32";
- return Status::NotImplemented(ss.str());
- }
- return Status::OK();
-}
-
-static Status FromInt64(const PrimitiveNode& node, TypePtr* out) {
- switch (node.logical_type()) {
- case LogicalType::NONE:
- *out = ::arrow::int64();
- break;
- case LogicalType::INT_64:
- *out = ::arrow::int64();
- break;
- case LogicalType::UINT_64:
- *out = ::arrow::uint64();
- break;
- case LogicalType::DECIMAL:
- *out = MakeDecimal128Type(node);
- break;
- case LogicalType::TIMESTAMP_MILLIS:
- *out = TIMESTAMP_MS;
- break;
- case LogicalType::TIMESTAMP_MICROS:
- *out = TIMESTAMP_US;
- break;
- case LogicalType::TIME_MICROS:
- *out = ::arrow::time64(::arrow::TimeUnit::MICRO);
- break;
- default:
- std::stringstream ss;
- ss << "Unhandled logical type " << LogicalTypeToString(node.logical_type())
- << " for INT64";
- return Status::NotImplemented(ss.str());
- }
- return Status::OK();
-}
-
-Status FromPrimitive(const PrimitiveNode& primitive, TypePtr* out) {
- if (primitive.logical_type() == LogicalType::NA) {
- *out = ::arrow::null();
- return Status::OK();
- }
-
- switch (primitive.physical_type()) {
- case ParquetType::BOOLEAN:
- *out = ::arrow::boolean();
- break;
- case ParquetType::INT32:
- RETURN_NOT_OK(FromInt32(primitive, out));
- break;
- case ParquetType::INT64:
- RETURN_NOT_OK(FromInt64(primitive, out));
- break;
- case ParquetType::INT96:
- *out = TIMESTAMP_NS;
- break;
- case ParquetType::FLOAT:
- *out = ::arrow::float32();
- break;
- case ParquetType::DOUBLE:
- *out = ::arrow::float64();
- break;
- case ParquetType::BYTE_ARRAY:
- RETURN_NOT_OK(FromByteArray(primitive, out));
- break;
- case ParquetType::FIXED_LEN_BYTE_ARRAY:
- RETURN_NOT_OK(FromFLBA(primitive, out));
- break;
- }
- return Status::OK();
-}
-
-// Forward declaration
-Status NodeToFieldInternal(const Node& node,
- const std::unordered_set<const Node*>* included_leaf_nodes,
- std::shared_ptr<Field>* out);
-
-/*
- * Auxilary function to test if a parquet schema node is a leaf node
- * that should be included in a resulting arrow schema
- */
-inline bool IsIncludedLeaf(const Node& node,
- const std::unordered_set<const Node*>* included_leaf_nodes) {
- if (included_leaf_nodes == nullptr) {
- return true;
- }
- auto search = included_leaf_nodes->find(&node);
- return (search != included_leaf_nodes->end());
-}
-
-Status StructFromGroup(const GroupNode& group,
- const std::unordered_set<const Node*>* included_leaf_nodes,
- TypePtr* out) {
- std::vector<std::shared_ptr<Field>> fields;
- std::shared_ptr<Field> field;
-
- *out = nullptr;
-
- for (int i = 0; i < group.field_count(); i++) {
- RETURN_NOT_OK(NodeToFieldInternal(*group.field(i), included_leaf_nodes, &field));
- if (field != nullptr) {
- fields.push_back(field);
- }
- }
- if (fields.size() > 0) {
- *out = std::make_shared<::arrow::StructType>(fields);
- }
- return Status::OK();
-}
-
-Status NodeToList(const GroupNode& group,
- const std::unordered_set<const Node*>* included_leaf_nodes,
- TypePtr* out) {
- *out = nullptr;
- if (group.field_count() == 1) {
- // This attempts to resolve the preferred 3-level list encoding.
- const Node& list_node = *group.field(0);
- if (list_node.is_group() && list_node.is_repeated()) {
- const auto& list_group = static_cast<const GroupNode&>(list_node);
- // Special case mentioned in the format spec:
- // If the name is array or ends in _tuple, this should be a list of struct
- // even for single child elements.
- if (list_group.field_count() == 1 && !HasStructListName(list_group)) {
- // List of primitive type
- std::shared_ptr<Field> item_field;
- RETURN_NOT_OK(
- NodeToFieldInternal(*list_group.field(0), included_leaf_nodes, &item_field));
-
- if (item_field != nullptr) {
- *out = ::arrow::list(item_field);
- }
- } else {
- // List of struct
- std::shared_ptr<::arrow::DataType> inner_type;
- RETURN_NOT_OK(StructFromGroup(list_group, included_leaf_nodes, &inner_type));
- if (inner_type != nullptr) {
- auto item_field = std::make_shared<Field>(list_node.name(), inner_type, false);
- *out = ::arrow::list(item_field);
- }
- }
- } else if (list_node.is_repeated()) {
- // repeated primitive node
- std::shared_ptr<::arrow::DataType> inner_type;
- if (IsIncludedLeaf(static_cast<const Node&>(list_node), included_leaf_nodes)) {
- RETURN_NOT_OK(
- FromPrimitive(static_cast<const PrimitiveNode&>(list_node), &inner_type));
- auto item_field = std::make_shared<Field>(list_node.name(), inner_type, false);
- *out = ::arrow::list(item_field);
- }
- } else {
- return Status::NotImplemented(
- "Non-repeated groups in a LIST-annotated group are not supported.");
- }
- } else {
- return Status::NotImplemented(
- "Only LIST-annotated groups with a single child can be handled.");
- }
- return Status::OK();
-}
-
-Status NodeToField(const Node& node, std::shared_ptr<Field>* out) {
- return NodeToFieldInternal(node, nullptr, out);
-}
-
-Status NodeToFieldInternal(const Node& node,
- const std::unordered_set<const Node*>* included_leaf_nodes,
- std::shared_ptr<Field>* out) {
- std::shared_ptr<::arrow::DataType> type = nullptr;
- bool nullable = !node.is_required();
-
- *out = nullptr;
-
- if (node.is_repeated()) {
- // 1-level LIST encoding fields are required
- std::shared_ptr<::arrow::DataType> inner_type;
- if (node.is_group()) {
- RETURN_NOT_OK(StructFromGroup(static_cast<const GroupNode&>(node),
- included_leaf_nodes, &inner_type));
- } else if (IsIncludedLeaf(node, included_leaf_nodes)) {
- RETURN_NOT_OK(FromPrimitive(static_cast<const PrimitiveNode&>(node), &inner_type));
- }
- if (inner_type != nullptr) {
- auto item_field = std::make_shared<Field>(node.name(), inner_type, false);
- type = ::arrow::list(item_field);
- nullable = false;
- }
- } else if (node.is_group()) {
- const auto& group = static_cast<const GroupNode&>(node);
- if (node.logical_type() == LogicalType::LIST) {
- RETURN_NOT_OK(NodeToList(group, included_leaf_nodes, &type));
- } else {
- RETURN_NOT_OK(StructFromGroup(group, included_leaf_nodes, &type));
- }
- } else {
- // Primitive (leaf) node
- if (IsIncludedLeaf(node, included_leaf_nodes)) {
- RETURN_NOT_OK(FromPrimitive(static_cast<const PrimitiveNode&>(node), &type));
- }
- }
- if (type != nullptr) {
- *out = std::make_shared<Field>(node.name(), type, nullable);
- }
- return Status::OK();
-}
-
-Status FromParquetSchema(
- const SchemaDescriptor* parquet_schema,
- const std::shared_ptr<const KeyValueMetadata>& key_value_metadata,
- std::shared_ptr<::arrow::Schema>* out) {
- const GroupNode& schema_node = *parquet_schema->group_node();
-
- int num_fields = static_cast<int>(schema_node.field_count());
- std::vector<std::shared_ptr<Field>> fields(num_fields);
- for (int i = 0; i < num_fields; i++) {
- RETURN_NOT_OK(NodeToField(*schema_node.field(i), &fields[i]));
- }
-
- *out = std::make_shared<::arrow::Schema>(fields, key_value_metadata);
- return Status::OK();
-}
-
-Status FromParquetSchema(
- const SchemaDescriptor* parquet_schema, const std::vector<int>& column_indices,
- const std::shared_ptr<const KeyValueMetadata>& key_value_metadata,
- std::shared_ptr<::arrow::Schema>* out) {
- // TODO(wesm): Consider adding an arrow::Schema name attribute, which comes
- // from the root Parquet node
-
- // Put the right leaf nodes in an unordered set
- // Index in column_indices should be unique, duplicate indices are merged into one and
- // ordering by its first appearing.
- int num_columns = static_cast<int>(column_indices.size());
- std::unordered_set<const Node*> top_nodes; // to deduplicate the top nodes
- std::vector<const Node*> base_nodes; // to keep the ordering
- std::unordered_set<const Node*> included_leaf_nodes(num_columns);
- for (int i = 0; i < num_columns; i++) {
- const ColumnDescriptor* column_desc = parquet_schema->Column(column_indices[i]);
- included_leaf_nodes.insert(column_desc->schema_node().get());
- const Node* column_root = parquet_schema->GetColumnRoot(column_indices[i]);
- auto insertion = top_nodes.insert(column_root);
- if (insertion.second) {
- base_nodes.push_back(column_root);
- }
- }
-
- std::vector<std::shared_ptr<Field>> fields;
- std::shared_ptr<Field> field;
- for (auto node : base_nodes) {
- RETURN_NOT_OK(NodeToFieldInternal(*node, &included_leaf_nodes, &field));
- if (field != nullptr) {
- fields.push_back(field);
- }
- }
-
- *out = std::make_shared<::arrow::Schema>(fields, key_value_metadata);
- return Status::OK();
-}
-
-Status FromParquetSchema(const SchemaDescriptor* parquet_schema,
- const std::vector<int>& column_indices,
- std::shared_ptr<::arrow::Schema>* out) {
- return FromParquetSchema(parquet_schema, column_indices, nullptr, out);
-}
-
-Status FromParquetSchema(const SchemaDescriptor* parquet_schema,
- std::shared_ptr<::arrow::Schema>* out) {
- return FromParquetSchema(parquet_schema, nullptr, out);
-}
-
-Status ListToNode(const std::shared_ptr<::arrow::ListType>& type, const std::string& name,
- bool nullable, const WriterProperties& properties,
- const ArrowWriterProperties& arrow_properties, NodePtr* out) {
- Repetition::type repetition = nullable ? Repetition::OPTIONAL : Repetition::REQUIRED;
-
- NodePtr element;
- RETURN_NOT_OK(FieldToNode(type->value_field(), properties, arrow_properties, &element));
-
- NodePtr list = GroupNode::Make("list", Repetition::REPEATED, {element});
- *out = GroupNode::Make(name, repetition, {list}, LogicalType::LIST);
- return Status::OK();
-}
-
-Status StructToNode(const std::shared_ptr<::arrow::StructType>& type,
- const std::string& name, bool nullable,
- const WriterProperties& properties,
- const ArrowWriterProperties& arrow_properties, NodePtr* out) {
- Repetition::type repetition = nullable ? Repetition::OPTIONAL : Repetition::REQUIRED;
-
- std::vector<NodePtr> children(type->num_children());
- for (int i = 0; i < type->num_children(); i++) {
- RETURN_NOT_OK(
- FieldToNode(type->child(i), properties, arrow_properties, &children[i]));
- }
-
- *out = GroupNode::Make(name, repetition, children);
- return Status::OK();
-}
-
-static Status GetTimestampMetadata(const ::arrow::TimestampType& type,
- const ArrowWriterProperties& properties,
- ParquetType::type* physical_type,
- LogicalType::type* logical_type) {
- auto unit = type.unit();
- *physical_type = ParquetType::INT64;
-
- if (properties.coerce_timestamps_enabled()) {
- auto coerce_unit = properties.coerce_timestamps_unit();
- if (coerce_unit == ::arrow::TimeUnit::MILLI) {
- *logical_type = LogicalType::TIMESTAMP_MILLIS;
- } else if (coerce_unit == ::arrow::TimeUnit::MICRO) {
- *logical_type = LogicalType::TIMESTAMP_MICROS;
- } else {
- return Status::NotImplemented(
- "Can only coerce Arrow timestamps to milliseconds"
- " or microseconds");
- }
- return Status::OK();
- }
-
- if (unit == ::arrow::TimeUnit::MILLI) {
- *logical_type = LogicalType::TIMESTAMP_MILLIS;
- } else if (unit == ::arrow::TimeUnit::MICRO) {
- *logical_type = LogicalType::TIMESTAMP_MICROS;
- } else if (unit == ::arrow::TimeUnit::NANO) {
- if (properties.support_deprecated_int96_timestamps()) {
- *physical_type = ParquetType::INT96;
- // No corresponding logical type
- } else {
- *logical_type = LogicalType::TIMESTAMP_MICROS;
- }
- } else {
- return Status::NotImplemented(
- "Only MILLI, MICRO, and NANOS units supported for Arrow timestamps with "
- "Parquet.");
- }
- return Status::OK();
-}
-
-Status FieldToNode(const std::shared_ptr<Field>& field,
- const WriterProperties& properties,
- const ArrowWriterProperties& arrow_properties, NodePtr* out) {
- LogicalType::type logical_type = LogicalType::NONE;
- ParquetType::type type;
- Repetition::type repetition =
- field->nullable() ? Repetition::OPTIONAL : Repetition::REQUIRED;
-
- int length = -1;
- int precision = -1;
- int scale = -1;
-
- switch (field->type()->id()) {
- case ArrowType::NA:
- type = ParquetType::INT32;
- logical_type = LogicalType::NA;
- break;
- case ArrowType::BOOL:
- type = ParquetType::BOOLEAN;
- break;
- case ArrowType::UINT8:
- type = ParquetType::INT32;
- logical_type = LogicalType::UINT_8;
- break;
- case ArrowType::INT8:
- type = ParquetType::INT32;
- logical_type = LogicalType::INT_8;
- break;
- case ArrowType::UINT16:
- type = ParquetType::INT32;
- logical_type = LogicalType::UINT_16;
- break;
- case ArrowType::INT16:
- type = ParquetType::INT32;
- logical_type = LogicalType::INT_16;
- break;
- case ArrowType::UINT32:
- if (properties.version() == ::parquet::ParquetVersion::PARQUET_1_0) {
- type = ParquetType::INT64;
- } else {
- type = ParquetType::INT32;
- logical_type = LogicalType::UINT_32;
- }
- break;
- case ArrowType::INT32:
- type = ParquetType::INT32;
- break;
- case ArrowType::UINT64:
- type = ParquetType::INT64;
- logical_type = LogicalType::UINT_64;
- break;
- case ArrowType::INT64:
- type = ParquetType::INT64;
- break;
- case ArrowType::FLOAT:
- type = ParquetType::FLOAT;
- break;
- case ArrowType::DOUBLE:
- type = ParquetType::DOUBLE;
- break;
- case ArrowType::STRING:
- type = ParquetType::BYTE_ARRAY;
- logical_type = LogicalType::UTF8;
- break;
- case ArrowType::BINARY:
- type = ParquetType::BYTE_ARRAY;
- break;
- case ArrowType::FIXED_SIZE_BINARY: {
- type = ParquetType::FIXED_LEN_BYTE_ARRAY;
- const auto& fixed_size_binary_type =
- static_cast<const ::arrow::FixedSizeBinaryType&>(*field->type());
- length = fixed_size_binary_type.byte_width();
- } break;
- case ArrowType::DECIMAL: {
- type = ParquetType::FIXED_LEN_BYTE_ARRAY;
- logical_type = LogicalType::DECIMAL;
- const auto& decimal_type =
- static_cast<const ::arrow::Decimal128Type&>(*field->type());
- precision = decimal_type.precision();
- scale = decimal_type.scale();
- length = DecimalSize(precision);
- } break;
- case ArrowType::DATE32:
- type = ParquetType::INT32;
- logical_type = LogicalType::DATE;
- break;
- case ArrowType::DATE64:
- type = ParquetType::INT32;
- logical_type = LogicalType::DATE;
- break;
- case ArrowType::TIMESTAMP:
- RETURN_NOT_OK(
- GetTimestampMetadata(static_cast<::arrow::TimestampType&>(*field->type()),
- arrow_properties, &type, &logical_type));
- break;
- case ArrowType::TIME32:
- type = ParquetType::INT32;
- logical_type = LogicalType::TIME_MILLIS;
- break;
- case ArrowType::TIME64: {
- auto time_type = static_cast<::arrow::Time64Type*>(field->type().get());
- if (time_type->unit() == ::arrow::TimeUnit::NANO) {
- return Status::NotImplemented("Nanosecond time not supported in Parquet.");
- }
- type = ParquetType::INT64;
- logical_type = LogicalType::TIME_MICROS;
- } break;
- case ArrowType::STRUCT: {
- auto struct_type = std::static_pointer_cast<::arrow::StructType>(field->type());
- return StructToNode(struct_type, field->name(), field->nullable(), properties,
- arrow_properties, out);
- }
- case ArrowType::LIST: {
- auto list_type = std::static_pointer_cast<::arrow::ListType>(field->type());
- return ListToNode(list_type, field->name(), field->nullable(), properties,
- arrow_properties, out);
- }
- case ArrowType::DICTIONARY: {
- // Parquet has no Dictionary type, dictionary-encoded is handled on
- // the encoding, not the schema level.
- const ::arrow::DictionaryType& dict_type =
- static_cast<const ::arrow::DictionaryType&>(*field->type());
- std::shared_ptr<::arrow::Field> unpacked_field =
- ::arrow::field(field->name(), dict_type.dictionary()->type(), field->nullable(),
- field->metadata());
- return FieldToNode(unpacked_field, properties, arrow_properties, out);
- }
- default: {
- // TODO: DENSE_UNION, SPARE_UNION, JSON_SCALAR, DECIMAL_TEXT, VARCHAR
- std::stringstream ss;
- ss << "Unhandled type for Arrow to Parquet schema conversion: ";
- ss << field->type()->ToString();
- return Status::NotImplemented(ss.str());
- }
- }
- PARQUET_CATCH_NOT_OK(*out =
- PrimitiveNode::Make(field->name(), repetition, type,
- logical_type, length, precision, scale));
- return Status::OK();
-}
-
-Status ToParquetSchema(const ::arrow::Schema* arrow_schema,
- const WriterProperties& properties,
- const ArrowWriterProperties& arrow_properties,
- std::shared_ptr<SchemaDescriptor>* out) {
- std::vector<NodePtr> nodes(arrow_schema->num_fields());
- for (int i = 0; i < arrow_schema->num_fields(); i++) {
- RETURN_NOT_OK(
- FieldToNode(arrow_schema->field(i), properties, arrow_properties, &nodes[i]));
- }
-
- NodePtr schema = GroupNode::Make("schema", Repetition::REQUIRED, nodes);
- *out = std::make_shared<::parquet::SchemaDescriptor>();
- PARQUET_CATCH_NOT_OK((*out)->Init(schema));
-
- return Status::OK();
-}
-
-Status ToParquetSchema(const ::arrow::Schema* arrow_schema,
- const WriterProperties& properties,
- std::shared_ptr<SchemaDescriptor>* out) {
- return ToParquetSchema(arrow_schema, properties, *default_arrow_writer_properties(),
- out);
-}
-
-/// \brief Compute the number of bytes required to represent a decimal of a
-/// given precision. Taken from the Apache Impala codebase. The comments next
-/// to the return values are the maximum value that can be represented in 2's
-/// complement with the returned number of bytes.
-int32_t DecimalSize(int32_t precision) {
- DCHECK_GE(precision, 1) << "decimal precision must be greater than or equal to 1, got "
- << precision;
- DCHECK_LE(precision, 38) << "decimal precision must be less than or equal to 38, got "
- << precision;
-
- switch (precision) {
- case 1:
- case 2:
- return 1; // 127
- case 3:
- case 4:
- return 2; // 32,767
- case 5:
- case 6:
- return 3; // 8,388,607
- case 7:
- case 8:
- case 9:
- return 4; // 2,147,483,427
- case 10:
- case 11:
- return 5; // 549,755,813,887
- case 12:
- case 13:
- case 14:
- return 6; // 140,737,488,355,327
- case 15:
- case 16:
- return 7; // 36,028,797,018,963,967
- case 17:
- case 18:
- return 8; // 9,223,372,036,854,775,807
- case 19:
- case 20:
- case 21:
- return 9; // 2,361,183,241,434,822,606,847
- case 22:
- case 23:
- return 10; // 604,462,909,807,314,587,353,087
- case 24:
- case 25:
- case 26:
- return 11; // 154,742,504,910,672,534,362,390,527
- case 27:
- case 28:
- return 12; // 39,614,081,257,132,168,796,771,975,167
- case 29:
- case 30:
- case 31:
- return 13; // 10,141,204,801,825,835,211,973,625,643,007
- case 32:
- case 33:
- return 14; // 2,596,148,429,267,413,814,265,248,164,610,047
- case 34:
- case 35:
- return 15; // 664,613,997,892,457,936,451,903,530,140,172,287
- case 36:
- case 37:
- case 38:
- return 16; // 170,141,183,460,469,231,731,687,303,715,884,105,727
- default:
- break;
- }
- DCHECK(false);
- return -1;
-}
-
-} // namespace arrow
-} // namespace parquet
diff --git a/src/parquet/arrow/schema.h b/src/parquet/arrow/schema.h
deleted file mode 100644
index 3b212da..0000000
--- a/src/parquet/arrow/schema.h
+++ /dev/null
@@ -1,94 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#ifndef PARQUET_ARROW_SCHEMA_H
-#define PARQUET_ARROW_SCHEMA_H
-
-#include <memory>
-#include <vector>
-
-#include "arrow/api.h"
-
-#include "parquet/api/schema.h"
-#include "parquet/api/writer.h"
-#include "parquet/arrow/writer.h"
-
-namespace arrow {
-
-class Status;
-
-} // namespace arrow
-
-namespace parquet {
-
-namespace arrow {
-
-PARQUET_EXPORT
-::arrow::Status NodeToField(const schema::Node& node,
- std::shared_ptr<::arrow::Field>* out);
-
-/// Convert parquet schema to arrow schema with selected indices
-/// \param parquet_schema to be converted
-/// \param column_indices indices of leaf nodes in parquet schema tree. Appearing ordering
-/// matters for the converted schema. Repeated indices are ignored
-/// except for the first one
-/// \param key_value_metadata optional metadata, can be nullptr
-/// \param out the corresponding arrow schema
-/// \return Status::OK() on a successful conversion.
-PARQUET_EXPORT
-::arrow::Status FromParquetSchema(
- const SchemaDescriptor* parquet_schema, const std::vector<int>& column_indices,
- const std::shared_ptr<const KeyValueMetadata>& key_value_metadata,
- std::shared_ptr<::arrow::Schema>* out);
-
-// Without indices
-PARQUET_EXPORT
-::arrow::Status FromParquetSchema(
- const SchemaDescriptor* parquet_schema,
- const std::shared_ptr<const KeyValueMetadata>& key_value_metadata,
- std::shared_ptr<::arrow::Schema>* out);
-
-// Without metadata
-::arrow::Status PARQUET_EXPORT FromParquetSchema(const SchemaDescriptor* parquet_schema,
- const std::vector<int>& column_indices,
- std::shared_ptr<::arrow::Schema>* out);
-
-// Without metadata or indices
-::arrow::Status PARQUET_EXPORT FromParquetSchema(const SchemaDescriptor* parquet_schema,
- std::shared_ptr<::arrow::Schema>* out);
-
-::arrow::Status PARQUET_EXPORT FieldToNode(const std::shared_ptr<::arrow::Field>& field,
- const WriterProperties& properties,
- const ArrowWriterProperties& arrow_properties,
- schema::NodePtr* out);
-
-::arrow::Status PARQUET_EXPORT
-ToParquetSchema(const ::arrow::Schema* arrow_schema, const WriterProperties& properties,
- const ArrowWriterProperties& arrow_properties,
- std::shared_ptr<SchemaDescriptor>* out);
-
-::arrow::Status PARQUET_EXPORT ToParquetSchema(const ::arrow::Schema* arrow_schema,
- const WriterProperties& properties,
- std::shared_ptr<SchemaDescriptor>* out);
-
-int32_t DecimalSize(int32_t precision);
-
-} // namespace arrow
-
-} // namespace parquet
-
-#endif // PARQUET_ARROW_SCHEMA_H
diff --git a/src/parquet/arrow/test-util.h b/src/parquet/arrow/test-util.h
deleted file mode 100644
index 631bb71..0000000
--- a/src/parquet/arrow/test-util.h
+++ /dev/null
@@ -1,525 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#include <limits>
-#include <string>
-#include <vector>
-
-#include "arrow/api.h"
-#include "arrow/test-util.h"
-#include "arrow/type_traits.h"
-#include "arrow/util/decimal.h"
-
-#include "parquet/arrow/record_reader.h"
-
-namespace arrow {
-// PARQUET-1382: backwards-compatible shim for arrow::test namespace
-namespace test {}
-
-using namespace ::arrow::test; // NOLINT
-
-} // namespace arrow
-
-namespace parquet {
-
-using internal::RecordReader;
-
-namespace arrow {
-
-using ::arrow::Array;
-using ::arrow::Status;
-
-template <int32_t PRECISION>
-struct DecimalWithPrecisionAndScale {
- static_assert(PRECISION >= 1 && PRECISION <= 38, "Invalid precision value");
-
- using type = ::arrow::Decimal128Type;
- static constexpr ::arrow::Type::type type_id = ::arrow::Decimal128Type::type_id;
- static constexpr int32_t precision = PRECISION;
- static constexpr int32_t scale = PRECISION - 1;
-};
-
-template <typename ArrowType>
-using is_arrow_float = std::is_floating_point<typename ArrowType::c_type>;
-
-template <typename ArrowType>
-using is_arrow_int = std::is_integral<typename ArrowType::c_type>;
-
-template <typename ArrowType>
-using is_arrow_date = std::is_same<ArrowType, ::arrow::Date64Type>;
-
-template <typename ArrowType>
-using is_arrow_string = std::is_same<ArrowType, ::arrow::StringType>;
-
-template <typename ArrowType>
-using is_arrow_binary = std::is_same<ArrowType, ::arrow::BinaryType>;
-
-template <typename ArrowType>
-using is_arrow_fixed_size_binary = std::is_same<ArrowType, ::arrow::FixedSizeBinaryType>;
-
-template <typename ArrowType>
-using is_arrow_bool = std::is_same<ArrowType, ::arrow::BooleanType>;
-
-template <class ArrowType>
-typename std::enable_if<is_arrow_float<ArrowType>::value, Status>::type NonNullArray(
- size_t size, std::shared_ptr<Array>* out) {
- using c_type = typename ArrowType::c_type;
- std::vector<c_type> values;
- ::arrow::random_real(size, 0, static_cast<c_type>(0), static_cast<c_type>(1), &values);
- ::arrow::NumericBuilder<ArrowType> builder;
- RETURN_NOT_OK(builder.AppendValues(values.data(), values.size()));
- return builder.Finish(out);
-}
-
-template <class ArrowType>
-typename std::enable_if<
- is_arrow_int<ArrowType>::value && !is_arrow_date<ArrowType>::value, Status>::type
-NonNullArray(size_t size, std::shared_ptr<Array>* out) {
- std::vector<typename ArrowType::c_type> values;
- ::arrow::randint(size, 0, 64, &values);
-
- // Passing data type so this will work with TimestampType too
- ::arrow::NumericBuilder<ArrowType> builder(std::make_shared<ArrowType>(),
- ::arrow::default_memory_pool());
- RETURN_NOT_OK(builder.AppendValues(values.data(), values.size()));
- return builder.Finish(out);
-}
-
-template <class ArrowType>
-typename std::enable_if<is_arrow_date<ArrowType>::value, Status>::type NonNullArray(
- size_t size, std::shared_ptr<Array>* out) {
- std::vector<typename ArrowType::c_type> values;
- ::arrow::randint(size, 0, 64, &values);
- for (size_t i = 0; i < size; i++) {
- values[i] *= 86400000;
- }
-
- // Passing data type so this will work with TimestampType too
- ::arrow::NumericBuilder<ArrowType> builder(std::make_shared<ArrowType>(),
- ::arrow::default_memory_pool());
- builder.AppendValues(values.data(), values.size());
- return builder.Finish(out);
-}
-
-template <class ArrowType>
-typename std::enable_if<
- is_arrow_string<ArrowType>::value || is_arrow_binary<ArrowType>::value, Status>::type
-NonNullArray(size_t size, std::shared_ptr<Array>* out) {
- using BuilderType = typename ::arrow::TypeTraits<ArrowType>::BuilderType;
- BuilderType builder;
- for (size_t i = 0; i < size; i++) {
- RETURN_NOT_OK(builder.Append("test-string"));
- }
- return builder.Finish(out);
-}
-
-template <typename ArrowType>
-typename std::enable_if<is_arrow_fixed_size_binary<ArrowType>::value, Status>::type
-NonNullArray(size_t size, std::shared_ptr<Array>* out) {
- using BuilderType = typename ::arrow::TypeTraits<ArrowType>::BuilderType;
- // set byte_width to the length of "fixed": 5
- // todo: find a way to generate test data with more diversity.
- BuilderType builder(::arrow::fixed_size_binary(5));
- for (size_t i = 0; i < size; i++) {
- RETURN_NOT_OK(builder.Append("fixed"));
- }
- return builder.Finish(out);
-}
-
-static inline void random_decimals(int64_t n, uint32_t seed, int32_t precision,
- uint8_t* out) {
- std::mt19937 gen(seed);
- std::uniform_int_distribution<uint32_t> d(0, std::numeric_limits<uint8_t>::max());
- const int32_t required_bytes = DecimalSize(precision);
- constexpr int32_t byte_width = 16;
- std::fill(out, out + byte_width * n, '\0');
-
- for (int64_t i = 0; i < n; ++i, out += byte_width) {
- std::generate(out, out + required_bytes,
- [&d, &gen] { return static_cast<uint8_t>(d(gen)); });
-
- // sign extend if the sign bit is set for the last byte generated
- // 0b10000000 == 0x80 == 128
- if ((out[required_bytes - 1] & '\x80') != 0) {
- std::fill(out + required_bytes, out + byte_width, '\xFF');
- }
- }
-}
-
-template <typename ArrowType, int32_t precision = ArrowType::precision>
-typename std::enable_if<
- std::is_same<ArrowType, DecimalWithPrecisionAndScale<precision>>::value, Status>::type
-NonNullArray(size_t size, std::shared_ptr<Array>* out) {
- constexpr int32_t kDecimalPrecision = precision;
- constexpr int32_t kDecimalScale = DecimalWithPrecisionAndScale<precision>::scale;
-
- const auto type = ::arrow::decimal(kDecimalPrecision, kDecimalScale);
- ::arrow::Decimal128Builder builder(type);
- const int32_t byte_width =
- static_cast<const ::arrow::Decimal128Type&>(*type).byte_width();
-
- constexpr int32_t seed = 0;
-
- std::shared_ptr<Buffer> out_buf;
- RETURN_NOT_OK(::arrow::AllocateBuffer(::arrow::default_memory_pool(), size * byte_width,
- &out_buf));
- random_decimals(size, seed, kDecimalPrecision, out_buf->mutable_data());
-
- RETURN_NOT_OK(builder.AppendValues(out_buf->data(), size));
- return builder.Finish(out);
-}
-
-template <class ArrowType>
-typename std::enable_if<is_arrow_bool<ArrowType>::value, Status>::type NonNullArray(
- size_t size, std::shared_ptr<Array>* out) {
- std::vector<uint8_t> values;
- ::arrow::randint(size, 0, 1, &values);
- ::arrow::BooleanBuilder builder;
- RETURN_NOT_OK(builder.AppendValues(values.data(), values.size()));
- return builder.Finish(out);
-}
-
-// This helper function only supports (size/2) nulls.
-template <typename ArrowType>
-typename std::enable_if<is_arrow_float<ArrowType>::value, Status>::type NullableArray(
- size_t size, size_t num_nulls, uint32_t seed, std::shared_ptr<Array>* out) {
- using c_type = typename ArrowType::c_type;
- std::vector<c_type> values;
- ::arrow::random_real(size, seed, static_cast<c_type>(-1e10), static_cast<c_type>(1e10),
- &values);
- std::vector<uint8_t> valid_bytes(size, 1);
-
- for (size_t i = 0; i < num_nulls; i++) {
- valid_bytes[i * 2] = 0;
- }
-
- ::arrow::NumericBuilder<ArrowType> builder;
- RETURN_NOT_OK(builder.AppendValues(values.data(), values.size(), valid_bytes.data()));
- return builder.Finish(out);
-}
-
-// This helper function only supports (size/2) nulls.
-template <typename ArrowType>
-typename std::enable_if<
- is_arrow_int<ArrowType>::value && !is_arrow_date<ArrowType>::value, Status>::type
-NullableArray(size_t size, size_t num_nulls, uint32_t seed, std::shared_ptr<Array>* out) {
- std::vector<typename ArrowType::c_type> values;
-
- // Seed is random in Arrow right now
- (void)seed;
- ::arrow::randint(size, 0, 64, &values);
- std::vector<uint8_t> valid_bytes(size, 1);
-
- for (size_t i = 0; i < num_nulls; i++) {
- valid_bytes[i * 2] = 0;
- }
-
- // Passing data type so this will work with TimestampType too
- ::arrow::NumericBuilder<ArrowType> builder(std::make_shared<ArrowType>(),
- ::arrow::default_memory_pool());
- RETURN_NOT_OK(builder.AppendValues(values.data(), values.size(), valid_bytes.data()));
- return builder.Finish(out);
-}
-
-template <typename ArrowType>
-typename std::enable_if<is_arrow_date<ArrowType>::value, Status>::type NullableArray(
- size_t size, size_t num_nulls, uint32_t seed, std::shared_ptr<Array>* out) {
- std::vector<typename ArrowType::c_type> values;
-
- // Seed is random in Arrow right now
- (void)seed;
- ::arrow::randint(size, 0, 64, &values);
- for (size_t i = 0; i < size; i++) {
- values[i] *= 86400000;
- }
- std::vector<uint8_t> valid_bytes(size, 1);
-
- for (size_t i = 0; i < num_nulls; i++) {
- valid_bytes[i * 2] = 0;
- }
-
- // Passing data type so this will work with TimestampType too
- ::arrow::NumericBuilder<ArrowType> builder(std::make_shared<ArrowType>(),
- ::arrow::default_memory_pool());
- builder.AppendValues(values.data(), values.size(), valid_bytes.data());
- return builder.Finish(out);
-}
-
-// This helper function only supports (size/2) nulls yet.
-template <typename ArrowType>
-typename std::enable_if<
- is_arrow_string<ArrowType>::value || is_arrow_binary<ArrowType>::value, Status>::type
-NullableArray(size_t size, size_t num_nulls, uint32_t seed,
- std::shared_ptr<::arrow::Array>* out) {
- std::vector<uint8_t> valid_bytes(size, 1);
-
- for (size_t i = 0; i < num_nulls; i++) {
- valid_bytes[i * 2] = 0;
- }
-
- using BuilderType = typename ::arrow::TypeTraits<ArrowType>::BuilderType;
- BuilderType builder;
-
- const int kBufferSize = 10;
- uint8_t buffer[kBufferSize];
- for (size_t i = 0; i < size; i++) {
- if (!valid_bytes[i]) {
- RETURN_NOT_OK(builder.AppendNull());
- } else {
- ::arrow::random_bytes(kBufferSize, seed + static_cast<uint32_t>(i), buffer);
- RETURN_NOT_OK(builder.Append(buffer, kBufferSize));
- }
- }
- return builder.Finish(out);
-}
-
-// This helper function only supports (size/2) nulls yet,
-// same as NullableArray<String|Binary>(..)
-template <typename ArrowType>
-typename std::enable_if<is_arrow_fixed_size_binary<ArrowType>::value, Status>::type
-NullableArray(size_t size, size_t num_nulls, uint32_t seed,
- std::shared_ptr<::arrow::Array>* out) {
- std::vector<uint8_t> valid_bytes(size, 1);
-
- for (size_t i = 0; i < num_nulls; i++) {
- valid_bytes[i * 2] = 0;
- }
-
- using BuilderType = typename ::arrow::TypeTraits<ArrowType>::BuilderType;
- const int byte_width = 10;
- BuilderType builder(::arrow::fixed_size_binary(byte_width));
-
- const int kBufferSize = byte_width;
- uint8_t buffer[kBufferSize];
- for (size_t i = 0; i < size; i++) {
- if (!valid_bytes[i]) {
- RETURN_NOT_OK(builder.AppendNull());
- } else {
- ::arrow::random_bytes(kBufferSize, seed + static_cast<uint32_t>(i), buffer);
- RETURN_NOT_OK(builder.Append(buffer));
- }
- }
- return builder.Finish(out);
-}
-
-template <typename ArrowType, int32_t precision = ArrowType::precision>
-typename std::enable_if<
- std::is_same<ArrowType, DecimalWithPrecisionAndScale<precision>>::value, Status>::type
-NullableArray(size_t size, size_t num_nulls, uint32_t seed,
- std::shared_ptr<::arrow::Array>* out) {
- std::vector<uint8_t> valid_bytes(size, '\1');
-
- for (size_t i = 0; i < num_nulls; ++i) {
- valid_bytes[i * 2] = '\0';
- }
-
- constexpr int32_t kDecimalPrecision = precision;
- constexpr int32_t kDecimalScale = DecimalWithPrecisionAndScale<precision>::scale;
- const auto type = ::arrow::decimal(kDecimalPrecision, kDecimalScale);
- const int32_t byte_width =
- static_cast<const ::arrow::Decimal128Type&>(*type).byte_width();
-
- std::shared_ptr<::arrow::Buffer> out_buf;
- RETURN_NOT_OK(::arrow::AllocateBuffer(::arrow::default_memory_pool(), size * byte_width,
- &out_buf));
-
- random_decimals(size, seed, precision, out_buf->mutable_data());
-
- ::arrow::Decimal128Builder builder(type);
- RETURN_NOT_OK(builder.AppendValues(out_buf->data(), size, valid_bytes.data()));
- return builder.Finish(out);
-}
-
-// This helper function only supports (size/2) nulls yet.
-template <class ArrowType>
-typename std::enable_if<is_arrow_bool<ArrowType>::value, Status>::type NullableArray(
- size_t size, size_t num_nulls, uint32_t seed, std::shared_ptr<Array>* out) {
- std::vector<uint8_t> values;
-
- // Seed is random in Arrow right now
- (void)seed;
-
- ::arrow::randint(size, 0, 1, &values);
- std::vector<uint8_t> valid_bytes(size, 1);
-
- for (size_t i = 0; i < num_nulls; i++) {
- valid_bytes[i * 2] = 0;
- }
-
- ::arrow::BooleanBuilder builder;
- RETURN_NOT_OK(builder.AppendValues(values.data(), values.size(), valid_bytes.data()));
- return builder.Finish(out);
-}
-
-/// Wrap an Array into a ListArray by splitting it up into size lists.
-///
-/// This helper function only supports (size/2) nulls.
-Status MakeListArray(const std::shared_ptr<Array>& values, int64_t size,
- int64_t null_count, bool nullable_values,
- std::shared_ptr<::arrow::ListArray>* out) {
- // We always include an empty list
- int64_t non_null_entries = size - null_count - 1;
- int64_t length_per_entry = values->length() / non_null_entries;
-
- auto offsets = AllocateBuffer();
- RETURN_NOT_OK(offsets->Resize((size + 1) * sizeof(int32_t)));
- int32_t* offsets_ptr = reinterpret_cast<int32_t*>(offsets->mutable_data());
-
- auto null_bitmap = AllocateBuffer();
- int64_t bitmap_size = ::arrow::BitUtil::BytesForBits(size);
- RETURN_NOT_OK(null_bitmap->Resize(bitmap_size));
- uint8_t* null_bitmap_ptr = null_bitmap->mutable_data();
- memset(null_bitmap_ptr, 0, bitmap_size);
-
- int32_t current_offset = 0;
- for (int64_t i = 0; i < size; i++) {
- offsets_ptr[i] = current_offset;
- if (!(((i % 2) == 0) && ((i / 2) < null_count))) {
- // Non-null list (list with index 1 is always empty).
- ::arrow::BitUtil::SetBit(null_bitmap_ptr, i);
- if (i != 1) {
- current_offset += static_cast<int32_t>(length_per_entry);
- }
- }
- }
- offsets_ptr[size] = static_cast<int32_t>(values->length());
-
- auto value_field = ::arrow::field("item", values->type(), nullable_values);
- *out = std::make_shared<::arrow::ListArray>(::arrow::list(value_field), size, offsets,
- values, null_bitmap, null_count);
-
- return Status::OK();
-}
-
-// Make an array containing only empty lists, with a null values array
-Status MakeEmptyListsArray(int64_t size, std::shared_ptr<Array>* out_array) {
- // Allocate an offsets buffer containing only zeroes
- std::shared_ptr<Buffer> offsets_buffer;
- const int64_t offsets_nbytes = (size + 1) * sizeof(int32_t);
- RETURN_NOT_OK(::arrow::AllocateBuffer(::arrow::default_memory_pool(), offsets_nbytes,
- &offsets_buffer));
- memset(offsets_buffer->mutable_data(), 0, offsets_nbytes);
-
- auto value_field =
- ::arrow::field("item", ::arrow::float64(), false /* nullable_values */);
- auto list_type = ::arrow::list(value_field);
-
- std::vector<std::shared_ptr<Buffer>> child_buffers = {nullptr /* null bitmap */,
- nullptr /* values */};
- auto child_data =
- ::arrow::ArrayData::Make(value_field->type(), 0, std::move(child_buffers));
-
- std::vector<std::shared_ptr<Buffer>> buffers = {nullptr /* bitmap */, offsets_buffer};
- auto array_data = ::arrow::ArrayData::Make(list_type, size, std::move(buffers));
- array_data->child_data.push_back(child_data);
-
- *out_array = ::arrow::MakeArray(array_data);
- return Status::OK();
-}
-
-static std::shared_ptr<::arrow::Column> MakeColumn(const std::string& name,
- const std::shared_ptr<Array>& array,
- bool nullable) {
- auto field = ::arrow::field(name, array->type(), nullable);
- return std::make_shared<::arrow::Column>(field, array);
-}
-
-static std::shared_ptr<::arrow::Column> MakeColumn(
- const std::string& name, const std::vector<std::shared_ptr<Array>>& arrays,
- bool nullable) {
- auto field = ::arrow::field(name, arrays[0]->type(), nullable);
- return std::make_shared<::arrow::Column>(field, arrays);
-}
-
-std::shared_ptr<::arrow::Table> MakeSimpleTable(const std::shared_ptr<Array>& values,
- bool nullable) {
- std::shared_ptr<::arrow::Column> column = MakeColumn("col", values, nullable);
- std::vector<std::shared_ptr<::arrow::Column>> columns({column});
- std::vector<std::shared_ptr<::arrow::Field>> fields({column->field()});
- auto schema = std::make_shared<::arrow::Schema>(fields);
- return ::arrow::Table::Make(schema, columns);
-}
-
-template <typename T>
-void ExpectArray(T* expected, Array* result) {
- auto p_array = static_cast<::arrow::PrimitiveArray*>(result);
- for (int i = 0; i < result->length(); i++) {
- EXPECT_EQ(expected[i], reinterpret_cast<const T*>(p_array->values()->data())[i]);
- }
-}
-
-template <typename ArrowType>
-void ExpectArrayT(void* expected, Array* result) {
- ::arrow::PrimitiveArray* p_array = static_cast<::arrow::PrimitiveArray*>(result);
- for (int64_t i = 0; i < result->length(); i++) {
- EXPECT_EQ(reinterpret_cast<typename ArrowType::c_type*>(expected)[i],
- reinterpret_cast<const typename ArrowType::c_type*>(
- p_array->values()->data())[i]);
- }
-}
-
-template <>
-void ExpectArrayT<::arrow::BooleanType>(void* expected, Array* result) {
- ::arrow::BooleanBuilder builder;
- EXPECT_OK(builder.AppendValues(reinterpret_cast<uint8_t*>(expected), result->length()));
-
- std::shared_ptr<Array> expected_array;
- EXPECT_OK(builder.Finish(&expected_array));
- EXPECT_TRUE(result->Equals(*expected_array));
-}
-
-template <typename ParquetType>
-void PrintBufferedLevels(const RecordReader& reader) {
- using T = typename ::parquet::type_traits<ParquetType::type_num>::value_type;
-
- const int16_t* def_levels = reader.def_levels();
- const int16_t* rep_levels = reader.rep_levels();
- const int64_t total_levels_read = reader.levels_position();
-
- const T* values = reinterpret_cast<const T*>(reader.values());
-
- std::cout << "def levels: ";
- for (int64_t i = 0; i < total_levels_read; ++i) {
- std::cout << def_levels[i] << " ";
- }
- std::cout << std::endl;
-
- std::cout << "rep levels: ";
- for (int64_t i = 0; i < total_levels_read; ++i) {
- std::cout << rep_levels[i] << " ";
- }
- std::cout << std::endl;
-
- std::cout << "values: ";
- for (int64_t i = 0; i < reader.values_written(); ++i) {
- std::cout << values[i] << " ";
- }
- std::cout << std::endl;
-}
-
-template <>
-void PrintBufferedLevels<ByteArrayType>(const RecordReader& reader) {}
-
-template <>
-void PrintBufferedLevels<FLBAType>(const RecordReader& reader) {}
-
-template <>
-void PrintBufferedLevels<Int96Type>(const RecordReader& reader) {}
-
-} // namespace arrow
-
-} // namespace parquet
diff --git a/src/parquet/arrow/writer.cc b/src/parquet/arrow/writer.cc
deleted file mode 100644
index b7d139e..0000000
--- a/src/parquet/arrow/writer.cc
+++ /dev/null
@@ -1,1151 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#include "parquet/arrow/writer.h"
-
-#include <algorithm>
-#include <string>
-#include <vector>
-
-#include "arrow/api.h"
-#include "arrow/compute/api.h"
-#include "arrow/util/bit-util.h"
-#include "arrow/visitor_inline.h"
-
-#include "parquet/arrow/schema.h"
-#include "parquet/util/logging.h"
-
-using arrow::Array;
-using arrow::BinaryArray;
-using arrow::BooleanArray;
-using arrow::ChunkedArray;
-using arrow::Decimal128Array;
-using arrow::Field;
-using arrow::FixedSizeBinaryArray;
-using arrow::Int16Array;
-using arrow::Int16Builder;
-using arrow::ListArray;
-using arrow::MemoryPool;
-using arrow::NumericArray;
-using arrow::PrimitiveArray;
-using arrow::ResizableBuffer;
-using arrow::Status;
-using arrow::Table;
-using arrow::TimeUnit;
-
-using arrow::compute::Cast;
-using arrow::compute::CastOptions;
-using arrow::compute::FunctionContext;
-
-using parquet::ParquetFileWriter;
-using parquet::ParquetVersion;
-using parquet::schema::GroupNode;
-
-namespace parquet {
-namespace arrow {
-
-namespace BitUtil = ::arrow::BitUtil;
-
-std::shared_ptr<ArrowWriterProperties> default_arrow_writer_properties() {
- static std::shared_ptr<ArrowWriterProperties> default_writer_properties =
- ArrowWriterProperties::Builder().build();
- return default_writer_properties;
-}
-
-namespace {
-
-class LevelBuilder {
- public:
- explicit LevelBuilder(MemoryPool* pool)
- : def_levels_(::arrow::int16(), pool), rep_levels_(::arrow::int16(), pool) {}
-
- Status VisitInline(const Array& array);
-
- template <typename T>
- typename std::enable_if<std::is_base_of<::arrow::FlatArray, T>::value, Status>::type
- Visit(const T& array) {
- array_offsets_.push_back(static_cast<int32_t>(array.offset()));
- valid_bitmaps_.push_back(array.null_bitmap_data());
- null_counts_.push_back(array.null_count());
- values_array_ = std::make_shared<T>(array.data());
- return Status::OK();
- }
-
- Status Visit(const ListArray& array) {
- array_offsets_.push_back(static_cast<int32_t>(array.offset()));
- valid_bitmaps_.push_back(array.null_bitmap_data());
- null_counts_.push_back(array.null_count());
- offsets_.push_back(array.raw_value_offsets());
-
- min_offset_idx_ = array.value_offset(min_offset_idx_);
- max_offset_idx_ = array.value_offset(max_offset_idx_);
-
- return VisitInline(*array.values());
- }
-
-#define NOT_IMPLEMENTED_VISIT(ArrowTypePrefix) \
- Status Visit(const ::arrow::ArrowTypePrefix##Array& array) { \
- return Status::NotImplemented("Level generation for " #ArrowTypePrefix \
- " not supported yet"); \
- }
-
- NOT_IMPLEMENTED_VISIT(Struct)
- NOT_IMPLEMENTED_VISIT(Union)
- NOT_IMPLEMENTED_VISIT(Dictionary)
-
- Status GenerateLevels(const Array& array, const std::shared_ptr<Field>& field,
- int64_t* values_offset, int64_t* num_values, int64_t* num_levels,
- const std::shared_ptr<ResizableBuffer>& def_levels_scratch,
- std::shared_ptr<Buffer>* def_levels_out,
- std::shared_ptr<Buffer>* rep_levels_out,
- std::shared_ptr<Array>* values_array) {
- // Work downwards to extract bitmaps and offsets
- min_offset_idx_ = 0;
- max_offset_idx_ = array.length();
- RETURN_NOT_OK(VisitInline(array));
- *num_values = max_offset_idx_ - min_offset_idx_;
- *values_offset = min_offset_idx_;
- *values_array = values_array_;
-
- // Walk downwards to extract nullability
- std::shared_ptr<Field> current_field = field;
- nullable_.push_back(current_field->nullable());
- while (current_field->type()->num_children() > 0) {
- if (current_field->type()->num_children() > 1) {
- return Status::NotImplemented(
- "Fields with more than one child are not supported.");
- } else {
- current_field = current_field->type()->child(0);
- }
- nullable_.push_back(current_field->nullable());
- }
-
- // Generate the levels.
- if (nullable_.size() == 1) {
- // We have a PrimitiveArray
- *rep_levels_out = nullptr;
- if (nullable_[0]) {
- RETURN_NOT_OK(
- def_levels_scratch->Resize(array.length() * sizeof(int16_t), false));
- auto def_levels_ptr =
- reinterpret_cast<int16_t*>(def_levels_scratch->mutable_data());
- if (array.null_count() == 0) {
- std::fill(def_levels_ptr, def_levels_ptr + array.length(), 1);
- } else if (array.null_count() == array.length()) {
- std::fill(def_levels_ptr, def_levels_ptr + array.length(), 0);
- } else {
- ::arrow::internal::BitmapReader valid_bits_reader(
- array.null_bitmap_data(), array.offset(), array.length());
- for (int i = 0; i < array.length(); i++) {
- def_levels_ptr[i] = valid_bits_reader.IsSet() ? 1 : 0;
- valid_bits_reader.Next();
- }
- }
-
- *def_levels_out = def_levels_scratch;
- } else {
- *def_levels_out = nullptr;
- }
- *num_levels = array.length();
- } else {
- RETURN_NOT_OK(rep_levels_.Append(0));
- RETURN_NOT_OK(HandleListEntries(0, 0, 0, array.length()));
-
- std::shared_ptr<Array> def_levels_array;
- std::shared_ptr<Array> rep_levels_array;
-
- RETURN_NOT_OK(def_levels_.Finish(&def_levels_array));
- RETURN_NOT_OK(rep_levels_.Finish(&rep_levels_array));
-
- *def_levels_out = static_cast<PrimitiveArray*>(def_levels_array.get())->values();
- *rep_levels_out = static_cast<PrimitiveArray*>(rep_levels_array.get())->values();
- *num_levels = rep_levels_array->length();
- }
-
- return Status::OK();
- }
-
- Status HandleList(int16_t def_level, int16_t rep_level, int64_t index) {
- if (nullable_[rep_level]) {
- if (null_counts_[rep_level] == 0 ||
- BitUtil::GetBit(valid_bitmaps_[rep_level], index + array_offsets_[rep_level])) {
- return HandleNonNullList(static_cast<int16_t>(def_level + 1), rep_level, index);
- } else {
- return def_levels_.Append(def_level);
- }
- } else {
- return HandleNonNullList(def_level, rep_level, index);
- }
- }
-
- Status HandleNonNullList(int16_t def_level, int16_t rep_level, int64_t index) {
- const int32_t inner_offset = offsets_[rep_level][index];
- const int32_t inner_length = offsets_[rep_level][index + 1] - inner_offset;
- const int64_t recursion_level = rep_level + 1;
- if (inner_length == 0) {
- return def_levels_.Append(def_level);
- }
- if (recursion_level < static_cast<int64_t>(offsets_.size())) {
- return HandleListEntries(static_cast<int16_t>(def_level + 1),
- static_cast<int16_t>(rep_level + 1), inner_offset,
- inner_length);
- } else {
- // We have reached the leaf: primitive list, handle remaining nullables
- const bool nullable_level = nullable_[recursion_level];
- const int64_t level_null_count = null_counts_[recursion_level];
- const uint8_t* level_valid_bitmap = valid_bitmaps_[recursion_level];
-
- for (int64_t i = 0; i < inner_length; i++) {
- if (i > 0) {
- RETURN_NOT_OK(rep_levels_.Append(static_cast<int16_t>(rep_level + 1)));
- }
- if (level_null_count && level_valid_bitmap == nullptr) {
- // Special case: this is a null array (all elements are null)
- RETURN_NOT_OK(def_levels_.Append(static_cast<int16_t>(def_level + 1)));
- } else if (nullable_level &&
- ((level_null_count == 0) ||
- BitUtil::GetBit(
- level_valid_bitmap,
- inner_offset + i + array_offsets_[recursion_level]))) {
- // Non-null element in a null level
- RETURN_NOT_OK(def_levels_.Append(static_cast<int16_t>(def_level + 2)));
- } else {
- // This can be produced in two case:
- // * elements are nullable and this one is null (i.e. max_def_level = def_level
- // + 2)
- // * elements are non-nullable (i.e. max_def_level = def_level + 1)
- RETURN_NOT_OK(def_levels_.Append(static_cast<int16_t>(def_level + 1)));
- }
- }
- return Status::OK();
- }
- }
-
- Status HandleListEntries(int16_t def_level, int16_t rep_level, int64_t offset,
- int64_t length) {
- for (int64_t i = 0; i < length; i++) {
- if (i > 0) {
- RETURN_NOT_OK(rep_levels_.Append(rep_level));
- }
- RETURN_NOT_OK(HandleList(def_level, rep_level, offset + i));
- }
- return Status::OK();
- }
-
- private:
- Int16Builder def_levels_;
- Int16Builder rep_levels_;
-
- std::vector<int64_t> null_counts_;
- std::vector<const uint8_t*> valid_bitmaps_;
- std::vector<const int32_t*> offsets_;
- std::vector<int32_t> array_offsets_;
- std::vector<bool> nullable_;
-
- int64_t min_offset_idx_;
- int64_t max_offset_idx_;
- std::shared_ptr<Array> values_array_;
-};
-
-Status LevelBuilder::VisitInline(const Array& array) {
- return VisitArrayInline(array, this);
-}
-
-struct ColumnWriterContext {
- ColumnWriterContext(MemoryPool* memory_pool, ArrowWriterProperties* properties)
- : memory_pool(memory_pool), properties(properties) {
- this->data_buffer = AllocateBuffer(memory_pool);
- this->def_levels_buffer = AllocateBuffer(memory_pool);
- }
-
- template <typename T>
- Status GetScratchData(const int64_t num_values, T** out) {
- RETURN_NOT_OK(this->data_buffer->Resize(num_values * sizeof(T), false));
- *out = reinterpret_cast<T*>(this->data_buffer->mutable_data());
- return Status::OK();
- }
-
- MemoryPool* memory_pool;
- ArrowWriterProperties* properties;
-
- // Buffer used for storing the data of an array converted to the physical type
- // as expected by parquet-cpp.
- std::shared_ptr<ResizableBuffer> data_buffer;
-
- // We use the shared ownership of this buffer
- std::shared_ptr<ResizableBuffer> def_levels_buffer;
-};
-
-Status GetLeafType(const ::arrow::DataType& type, ::arrow::Type::type* leaf_type) {
- if (type.id() == ::arrow::Type::LIST || type.id() == ::arrow::Type::STRUCT) {
- if (type.num_children() != 1) {
- return Status::Invalid("Nested column branch had multiple children");
- }
- return GetLeafType(*type.child(0)->type(), leaf_type);
- } else {
- *leaf_type = type.id();
- return Status::OK();
- }
-}
-
-class ArrowColumnWriter {
- public:
- ArrowColumnWriter(ColumnWriterContext* ctx, ColumnWriter* column_writer,
- const std::shared_ptr<Field>& field)
- : ctx_(ctx), writer_(column_writer), field_(field) {}
-
- Status Write(const Array& data);
-
- Status Write(const ChunkedArray& data, int64_t offset, const int64_t size) {
- int64_t absolute_position = 0;
- int chunk_index = 0;
- int64_t chunk_offset = 0;
- while (chunk_index < data.num_chunks() && absolute_position < offset) {
- const int64_t chunk_length = data.chunk(chunk_index)->length();
- if (absolute_position + chunk_length > offset) {
- // Relative offset into the chunk to reach the desired start offset for
- // writing
- chunk_offset = offset - absolute_position;
- break;
- } else {
- ++chunk_index;
- absolute_position += chunk_length;
- }
- }
-
- if (absolute_position >= data.length()) {
- return Status::Invalid("Cannot write data at offset past end of chunked array");
- }
-
- int64_t values_written = 0;
- while (values_written < size) {
- const Array& chunk = *data.chunk(chunk_index);
- const int64_t available_values = chunk.length() - chunk_offset;
- const int64_t chunk_write_size = std::min(size - values_written, available_values);
-
- // The chunk offset here will be 0 except for possibly the first chunk
- // because of the advancing logic above
- std::shared_ptr<Array> array_to_write = chunk.Slice(chunk_offset, chunk_write_size);
- RETURN_NOT_OK(Write(*array_to_write));
-
- if (chunk_write_size == available_values) {
- chunk_offset = 0;
- ++chunk_index;
- }
- values_written += chunk_write_size;
- }
-
- return Status::OK();
- }
-
- Status Close() {
- PARQUET_CATCH_NOT_OK(writer_->Close());
- return Status::OK();
- }
-
- private:
- template <typename ParquetType, typename ArrowType>
- Status TypedWriteBatch(const Array& data, int64_t num_levels, const int16_t* def_levels,
- const int16_t* rep_levels);
-
- Status WriteTimestamps(const Array& data, int64_t num_levels, const int16_t* def_levels,
- const int16_t* rep_levels);
-
- Status WriteTimestampsCoerce(const Array& data, int64_t num_levels,
- const int16_t* def_levels, const int16_t* rep_levels);
-
- template <typename ParquetType, typename ArrowType>
- Status WriteNonNullableBatch(const ArrowType& type, int64_t num_values,
- int64_t num_levels, const int16_t* def_levels,
- const int16_t* rep_levels,
- const typename ArrowType::c_type* values);
-
- template <typename ParquetType, typename ArrowType>
- Status WriteNullableBatch(const ArrowType& type, int64_t num_values, int64_t num_levels,
- const int16_t* def_levels, const int16_t* rep_levels,
- const uint8_t* valid_bits, int64_t valid_bits_offset,
- const typename ArrowType::c_type* values);
-
- template <typename ParquetType>
- Status WriteBatch(int64_t num_levels, const int16_t* def_levels,
- const int16_t* rep_levels,
- const typename ParquetType::c_type* values) {
- auto typed_writer = static_cast<TypedColumnWriter<ParquetType>*>(writer_);
- PARQUET_CATCH_NOT_OK(
- typed_writer->WriteBatch(num_levels, def_levels, rep_levels, values));
- return Status::OK();
- }
-
- template <typename ParquetType>
- Status WriteBatchSpaced(int64_t num_levels, const int16_t* def_levels,
- const int16_t* rep_levels, const uint8_t* valid_bits,
- int64_t valid_bits_offset,
- const typename ParquetType::c_type* values) {
- auto typed_writer = static_cast<TypedColumnWriter<ParquetType>*>(writer_);
- PARQUET_CATCH_NOT_OK(typed_writer->WriteBatchSpaced(
- num_levels, def_levels, rep_levels, valid_bits, valid_bits_offset, values));
- return Status::OK();
- }
-
- ColumnWriterContext* ctx_;
- ColumnWriter* writer_;
- std::shared_ptr<Field> field_;
-};
-
-template <typename ParquetType, typename ArrowType>
-Status ArrowColumnWriter::TypedWriteBatch(const Array& array, int64_t num_levels,
- const int16_t* def_levels,
- const int16_t* rep_levels) {
- using ArrowCType = typename ArrowType::c_type;
-
- const auto& data = static_cast<const PrimitiveArray&>(array);
- const ArrowCType* values = nullptr;
- // The values buffer may be null if the array is empty (ARROW-2744)
- if (data.values() != nullptr) {
- values = reinterpret_cast<const ArrowCType*>(data.values()->data()) + data.offset();
- } else {
- DCHECK_EQ(data.length(), 0);
- }
-
- if (writer_->descr()->schema_node()->is_required() || (data.null_count() == 0)) {
- // no nulls, just dump the data
- RETURN_NOT_OK((WriteNonNullableBatch<ParquetType, ArrowType>(
- static_cast<const ArrowType&>(*array.type()), array.length(), num_levels,
- def_levels, rep_levels, values)));
- } else {
- const uint8_t* valid_bits = data.null_bitmap_data();
- RETURN_NOT_OK((WriteNullableBatch<ParquetType, ArrowType>(
- static_cast<const ArrowType&>(*array.type()), data.length(), num_levels,
- def_levels, rep_levels, valid_bits, data.offset(), values)));
- }
- return Status::OK();
-}
-
-template <typename ParquetType, typename ArrowType>
-Status ArrowColumnWriter::WriteNonNullableBatch(
- const ArrowType& type, int64_t num_values, int64_t num_levels,
- const int16_t* def_levels, const int16_t* rep_levels,
- const typename ArrowType::c_type* values) {
- using ParquetCType = typename ParquetType::c_type;
- ParquetCType* buffer;
- RETURN_NOT_OK(ctx_->GetScratchData<ParquetCType>(num_values, &buffer));
-
- std::copy(values, values + num_values, buffer);
-
- return WriteBatch<ParquetType>(num_levels, def_levels, rep_levels, buffer);
-}
-
-template <>
-Status ArrowColumnWriter::WriteNonNullableBatch<Int32Type, ::arrow::Date64Type>(
- const ::arrow::Date64Type& type, int64_t num_values, int64_t num_levels,
- const int16_t* def_levels, const int16_t* rep_levels, const int64_t* values) {
- int32_t* buffer;
- RETURN_NOT_OK(ctx_->GetScratchData<int32_t>(num_levels, &buffer));
-
- for (int i = 0; i < num_values; i++) {
- buffer[i] = static_cast<int32_t>(values[i] / 86400000);
- }
-
- return WriteBatch<Int32Type>(num_levels, def_levels, rep_levels, buffer);
-}
-
-template <>
-Status ArrowColumnWriter::WriteNonNullableBatch<Int32Type, ::arrow::Time32Type>(
- const ::arrow::Time32Type& type, int64_t num_values, int64_t num_levels,
- const int16_t* def_levels, const int16_t* rep_levels, const int32_t* values) {
- int32_t* buffer;
- RETURN_NOT_OK(ctx_->GetScratchData<int32_t>(num_levels, &buffer));
- if (type.unit() == TimeUnit::SECOND) {
- for (int i = 0; i < num_values; i++) {
- buffer[i] = values[i] * 1000;
- }
- } else {
- std::copy(values, values + num_values, buffer);
- }
- return WriteBatch<Int32Type>(num_levels, def_levels, rep_levels, buffer);
-}
-
-#define NONNULLABLE_BATCH_FAST_PATH(ParquetType, ArrowType, CType) \
- template <> \
- Status ArrowColumnWriter::WriteNonNullableBatch<ParquetType, ArrowType>( \
- const ArrowType& type, int64_t num_values, int64_t num_levels, \
- const int16_t* def_levels, const int16_t* rep_levels, const CType* buffer) { \
- return WriteBatch<ParquetType>(num_levels, def_levels, rep_levels, buffer); \
- }
-
-NONNULLABLE_BATCH_FAST_PATH(Int32Type, ::arrow::Int32Type, int32_t)
-NONNULLABLE_BATCH_FAST_PATH(Int32Type, ::arrow::Date32Type, int32_t)
-NONNULLABLE_BATCH_FAST_PATH(Int64Type, ::arrow::Int64Type, int64_t)
-NONNULLABLE_BATCH_FAST_PATH(Int64Type, ::arrow::Time64Type, int64_t)
-NONNULLABLE_BATCH_FAST_PATH(FloatType, ::arrow::FloatType, float)
-NONNULLABLE_BATCH_FAST_PATH(DoubleType, ::arrow::DoubleType, double)
-
-template <typename ParquetType, typename ArrowType>
-Status ArrowColumnWriter::WriteNullableBatch(
- const ArrowType& type, int64_t num_values, int64_t num_levels,
- const int16_t* def_levels, const int16_t* rep_levels, const uint8_t* valid_bits,
- int64_t valid_bits_offset, const typename ArrowType::c_type* values) {
- using ParquetCType = typename ParquetType::c_type;
-
- ParquetCType* buffer;
- RETURN_NOT_OK(ctx_->GetScratchData<ParquetCType>(num_levels, &buffer));
- for (int i = 0; i < num_values; i++) {
- buffer[i] = static_cast<ParquetCType>(values[i]);
- }
-
- return WriteBatchSpaced<ParquetType>(num_levels, def_levels, rep_levels, valid_bits,
- valid_bits_offset, buffer);
-}
-
-template <>
-Status ArrowColumnWriter::WriteNullableBatch<Int32Type, ::arrow::Date64Type>(
- const ::arrow::Date64Type& type, int64_t num_values, int64_t num_levels,
- const int16_t* def_levels, const int16_t* rep_levels, const uint8_t* valid_bits,
- int64_t valid_bits_offset, const int64_t* values) {
- int32_t* buffer;
- RETURN_NOT_OK(ctx_->GetScratchData<int32_t>(num_values, &buffer));
-
- for (int i = 0; i < num_values; i++) {
- // Convert from milliseconds into days since the epoch
- buffer[i] = static_cast<int32_t>(values[i] / 86400000);
- }
-
- return WriteBatchSpaced<Int32Type>(num_levels, def_levels, rep_levels, valid_bits,
- valid_bits_offset, buffer);
-}
-
-template <>
-Status ArrowColumnWriter::WriteNullableBatch<Int32Type, ::arrow::Time32Type>(
- const ::arrow::Time32Type& type, int64_t num_values, int64_t num_levels,
- const int16_t* def_levels, const int16_t* rep_levels, const uint8_t* valid_bits,
- int64_t valid_bits_offset, const int32_t* values) {
- int32_t* buffer;
- RETURN_NOT_OK(ctx_->GetScratchData<int32_t>(num_values, &buffer));
-
- if (type.unit() == TimeUnit::SECOND) {
- for (int i = 0; i < num_values; i++) {
- buffer[i] = values[i] * 1000;
- }
- } else {
- for (int i = 0; i < num_values; i++) {
- buffer[i] = values[i];
- }
- }
- return WriteBatchSpaced<Int32Type>(num_levels, def_levels, rep_levels, valid_bits,
- valid_bits_offset, buffer);
-}
-
-#define NULLABLE_BATCH_FAST_PATH(ParquetType, ArrowType, CType) \
- template <> \
- Status ArrowColumnWriter::WriteNullableBatch<ParquetType, ArrowType>( \
- const ArrowType& type, int64_t num_values, int64_t num_levels, \
- const int16_t* def_levels, const int16_t* rep_levels, const uint8_t* valid_bits, \
- int64_t valid_bits_offset, const CType* values) { \
- return WriteBatchSpaced<ParquetType>(num_levels, def_levels, rep_levels, valid_bits, \
- valid_bits_offset, values); \
- }
-
-NULLABLE_BATCH_FAST_PATH(Int32Type, ::arrow::Int32Type, int32_t)
-NULLABLE_BATCH_FAST_PATH(Int32Type, ::arrow::Date32Type, int32_t)
-NULLABLE_BATCH_FAST_PATH(Int64Type, ::arrow::Int64Type, int64_t)
-NULLABLE_BATCH_FAST_PATH(Int64Type, ::arrow::Time64Type, int64_t)
-NULLABLE_BATCH_FAST_PATH(FloatType, ::arrow::FloatType, float)
-NULLABLE_BATCH_FAST_PATH(DoubleType, ::arrow::DoubleType, double)
-NULLABLE_BATCH_FAST_PATH(Int64Type, ::arrow::TimestampType, int64_t)
-NONNULLABLE_BATCH_FAST_PATH(Int64Type, ::arrow::TimestampType, int64_t)
-
-template <>
-Status ArrowColumnWriter::WriteNullableBatch<Int96Type, ::arrow::TimestampType>(
- const ::arrow::TimestampType& type, int64_t num_values, int64_t num_levels,
- const int16_t* def_levels, const int16_t* rep_levels, const uint8_t* valid_bits,
- int64_t valid_bits_offset, const int64_t* values) {
- Int96* buffer;
- RETURN_NOT_OK(ctx_->GetScratchData<Int96>(num_values, &buffer));
- if (type.unit() == TimeUnit::NANO) {
- for (int i = 0; i < num_values; i++) {
- internal::NanosecondsToImpalaTimestamp(values[i], &buffer[i]);
- }
- } else {
- return Status::NotImplemented("Only NANO timestamps are supported for Int96 writing");
- }
- return WriteBatchSpaced<Int96Type>(num_levels, def_levels, rep_levels, valid_bits,
- valid_bits_offset, buffer);
-}
-
-template <>
-Status ArrowColumnWriter::WriteNonNullableBatch<Int96Type, ::arrow::TimestampType>(
- const ::arrow::TimestampType& type, int64_t num_values, int64_t num_levels,
- const int16_t* def_levels, const int16_t* rep_levels, const int64_t* values) {
- Int96* buffer;
- RETURN_NOT_OK(ctx_->GetScratchData<Int96>(num_values, &buffer));
- if (type.unit() == TimeUnit::NANO) {
- for (int i = 0; i < num_values; i++) {
- internal::NanosecondsToImpalaTimestamp(values[i], buffer + i);
- }
- } else {
- return Status::NotImplemented("Only NANO timestamps are supported for Int96 writing");
- }
- return WriteBatch<Int96Type>(num_levels, def_levels, rep_levels, buffer);
-}
-
-Status ArrowColumnWriter::WriteTimestamps(const Array& values, int64_t num_levels,
- const int16_t* def_levels,
- const int16_t* rep_levels) {
- const auto& type = static_cast<const ::arrow::TimestampType&>(*values.type());
-
- const bool is_nanosecond = type.unit() == TimeUnit::NANO;
-
- // In the case where support_deprecated_int96_timestamps was specified
- // and coerce_timestamps_enabled was specified, a nanosecond column
- // will have a physical type of int64. In that case, we fall through
- // to the else if below.
- //
- // See https://issues.apache.org/jira/browse/ARROW-2082
- if (is_nanosecond && ctx_->properties->support_deprecated_int96_timestamps() &&
- !ctx_->properties->coerce_timestamps_enabled()) {
- return TypedWriteBatch<Int96Type, ::arrow::TimestampType>(values, num_levels,
- def_levels, rep_levels);
- } else if (is_nanosecond ||
- (ctx_->properties->coerce_timestamps_enabled() &&
- (type.unit() != ctx_->properties->coerce_timestamps_unit()))) {
- // Casting is required. This covers several cases
- // * Nanoseconds -> cast to microseconds
- // * coerce_timestamps_enabled_, cast all timestamps to requested unit
- return WriteTimestampsCoerce(values, num_levels, def_levels, rep_levels);
- } else {
- // No casting of timestamps is required, take the fast path
- return TypedWriteBatch<Int64Type, ::arrow::TimestampType>(values, num_levels,
- def_levels, rep_levels);
- }
-}
-
-Status ArrowColumnWriter::WriteTimestampsCoerce(const Array& array, int64_t num_levels,
- const int16_t* def_levels,
- const int16_t* rep_levels) {
- int64_t* buffer;
- RETURN_NOT_OK(ctx_->GetScratchData<int64_t>(num_levels, &buffer));
-
- const auto& data = static_cast<const ::arrow::TimestampArray&>(array);
-
- auto values = data.raw_values();
- const auto& type = static_cast<const ::arrow::TimestampType&>(*array.type());
-
- TimeUnit::type target_unit = ctx_->properties->coerce_timestamps_enabled()
- ? ctx_->properties->coerce_timestamps_unit()
- : TimeUnit::MICRO;
- auto target_type = ::arrow::timestamp(target_unit);
-
- auto DivideBy = [&](const int64_t factor) {
- for (int64_t i = 0; i < array.length(); i++) {
- if (!data.IsNull(i) && (values[i] % factor != 0)) {
- std::stringstream ss;
- ss << "Casting from " << type.ToString() << " to " << target_type->ToString()
- << " would lose data: " << values[i];
- return Status::Invalid(ss.str());
- }
- buffer[i] = values[i] / factor;
- }
- return Status::OK();
- };
-
- auto MultiplyBy = [&](const int64_t factor) {
- for (int64_t i = 0; i < array.length(); i++) {
- buffer[i] = values[i] * factor;
- }
- return Status::OK();
- };
-
- if (type.unit() == TimeUnit::NANO) {
- if (target_unit == TimeUnit::MICRO) {
- RETURN_NOT_OK(DivideBy(1000));
- } else {
- DCHECK_EQ(TimeUnit::MILLI, target_unit);
- RETURN_NOT_OK(DivideBy(1000000));
- }
- } else if (type.unit() == TimeUnit::SECOND) {
- RETURN_NOT_OK(MultiplyBy(target_unit == TimeUnit::MICRO ? 1000000 : 1000));
- } else if (type.unit() == TimeUnit::MILLI) {
- DCHECK_EQ(TimeUnit::MICRO, target_unit);
- RETURN_NOT_OK(MultiplyBy(1000));
- } else {
- DCHECK_EQ(TimeUnit::MILLI, target_unit);
- RETURN_NOT_OK(DivideBy(1000));
- }
-
- if (writer_->descr()->schema_node()->is_required() || (data.null_count() == 0)) {
- // no nulls, just dump the data
- RETURN_NOT_OK((WriteNonNullableBatch<Int64Type, ::arrow::TimestampType>(
- static_cast<const ::arrow::TimestampType&>(*target_type), array.length(),
- num_levels, def_levels, rep_levels, buffer)));
- } else {
- const uint8_t* valid_bits = data.null_bitmap_data();
- RETURN_NOT_OK((WriteNullableBatch<Int64Type, ::arrow::TimestampType>(
- static_cast<const ::arrow::TimestampType&>(*target_type), array.length(),
- num_levels, def_levels, rep_levels, valid_bits, data.offset(), buffer)));
- }
- return Status::OK();
-}
-
-// This specialization seems quite similar but it significantly differs in two points:
-// * offset is added at the most latest time to the pointer as we have sub-byte access
-// * Arrow data is stored bitwise thus we cannot use std::copy to transform from
-// ArrowType::c_type to ParquetType::c_type
-
-template <>
-Status ArrowColumnWriter::TypedWriteBatch<BooleanType, ::arrow::BooleanType>(
- const Array& array, int64_t num_levels, const int16_t* def_levels,
- const int16_t* rep_levels) {
- bool* buffer;
- RETURN_NOT_OK(ctx_->GetScratchData<bool>(array.length(), &buffer));
-
- const auto& data = static_cast<const BooleanArray&>(array);
- const uint8_t* values = nullptr;
- // The values buffer may be null if the array is empty (ARROW-2744)
- if (data.values() != nullptr) {
- values = reinterpret_cast<const uint8_t*>(data.values()->data());
- } else {
- DCHECK_EQ(data.length(), 0);
- }
-
- int buffer_idx = 0;
- int64_t offset = array.offset();
- for (int i = 0; i < data.length(); i++) {
- if (!data.IsNull(i)) {
- buffer[buffer_idx++] = BitUtil::GetBit(values, offset + i);
- }
- }
-
- return WriteBatch<BooleanType>(num_levels, def_levels, rep_levels, buffer);
-}
-
-template <>
-Status ArrowColumnWriter::TypedWriteBatch<Int32Type, ::arrow::NullType>(
- const Array& array, int64_t num_levels, const int16_t* def_levels,
- const int16_t* rep_levels) {
- return WriteBatch<Int32Type>(num_levels, def_levels, rep_levels, nullptr);
-}
-
-template <>
-Status ArrowColumnWriter::TypedWriteBatch<ByteArrayType, ::arrow::BinaryType>(
- const Array& array, int64_t num_levels, const int16_t* def_levels,
- const int16_t* rep_levels) {
- ByteArray* buffer;
- RETURN_NOT_OK(ctx_->GetScratchData<ByteArray>(num_levels, &buffer));
-
- const auto& data = static_cast<const BinaryArray&>(array);
-
- // In the case of an array consisting of only empty strings or all null,
- // data.data() points already to a nullptr, thus data.data()->data() will
- // segfault.
- const uint8_t* values = nullptr;
- if (data.value_data()) {
- values = reinterpret_cast<const uint8_t*>(data.value_data()->data());
- DCHECK(values != nullptr);
- }
-
- // Slice offset is accounted for in raw_value_offsets
- const int32_t* value_offset = data.raw_value_offsets();
-
- if (writer_->descr()->schema_node()->is_required() || (data.null_count() == 0)) {
- // no nulls, just dump the data
- for (int64_t i = 0; i < data.length(); i++) {
- buffer[i] =
- ByteArray(value_offset[i + 1] - value_offset[i], values + value_offset[i]);
- }
- } else {
- int buffer_idx = 0;
- for (int64_t i = 0; i < data.length(); i++) {
- if (!data.IsNull(i)) {
- buffer[buffer_idx++] =
- ByteArray(value_offset[i + 1] - value_offset[i], values + value_offset[i]);
- }
- }
- }
-
- return WriteBatch<ByteArrayType>(num_levels, def_levels, rep_levels, buffer);
-}
-
-template <>
-Status ArrowColumnWriter::TypedWriteBatch<FLBAType, ::arrow::FixedSizeBinaryType>(
- const Array& array, int64_t num_levels, const int16_t* def_levels,
- const int16_t* rep_levels) {
- const auto& data = static_cast<const FixedSizeBinaryArray&>(array);
- const int64_t length = data.length();
-
- FLBA* buffer;
- RETURN_NOT_OK(ctx_->GetScratchData<FLBA>(num_levels, &buffer));
-
- if (writer_->descr()->schema_node()->is_required() || data.null_count() == 0) {
- // no nulls, just dump the data
- // todo(advancedxy): use a writeBatch to avoid this step
- for (int64_t i = 0; i < length; i++) {
- buffer[i] = FixedLenByteArray(data.GetValue(i));
- }
- } else {
- int buffer_idx = 0;
- for (int64_t i = 0; i < length; i++) {
- if (!data.IsNull(i)) {
- buffer[buffer_idx++] = FixedLenByteArray(data.GetValue(i));
- }
- }
- }
-
- return WriteBatch<FLBAType>(num_levels, def_levels, rep_levels, buffer);
-}
-
-template <>
-Status ArrowColumnWriter::TypedWriteBatch<FLBAType, ::arrow::Decimal128Type>(
- const Array& array, int64_t num_levels, const int16_t* def_levels,
- const int16_t* rep_levels) {
- const auto& data = static_cast<const Decimal128Array&>(array);
- const int64_t length = data.length();
-
- FLBA* buffer;
- RETURN_NOT_OK(ctx_->GetScratchData<FLBA>(num_levels, &buffer));
-
- const auto& decimal_type = static_cast<const ::arrow::Decimal128Type&>(*data.type());
- const int32_t offset =
- decimal_type.byte_width() - DecimalSize(decimal_type.precision());
-
- const bool does_not_have_nulls =
- writer_->descr()->schema_node()->is_required() || data.null_count() == 0;
-
- const auto valid_value_count = static_cast<size_t>(length - data.null_count()) * 2;
- std::vector<uint64_t> big_endian_values(valid_value_count);
-
- // TODO(phillipc): Look into whether our compilers will perform loop unswitching so we
- // don't have to keep writing two loops to handle the case where we know there are no
- // nulls
- if (does_not_have_nulls) {
- // no nulls, just dump the data
- // todo(advancedxy): use a writeBatch to avoid this step
- for (int64_t i = 0, j = 0; i < length; ++i, j += 2) {
- auto unsigned_64_bit = reinterpret_cast<const uint64_t*>(data.GetValue(i));
- big_endian_values[j] = ::arrow::BitUtil::ToBigEndian(unsigned_64_bit[1]);
- big_endian_values[j + 1] = ::arrow::BitUtil::ToBigEndian(unsigned_64_bit[0]);
- buffer[i] = FixedLenByteArray(
- reinterpret_cast<const uint8_t*>(&big_endian_values[j]) + offset);
- }
- } else {
- for (int64_t i = 0, buffer_idx = 0, j = 0; i < length; ++i) {
- if (!data.IsNull(i)) {
- auto unsigned_64_bit = reinterpret_cast<const uint64_t*>(data.GetValue(i));
- big_endian_values[j] = ::arrow::BitUtil::ToBigEndian(unsigned_64_bit[1]);
- big_endian_values[j + 1] = ::arrow::BitUtil::ToBigEndian(unsigned_64_bit[0]);
- buffer[buffer_idx++] = FixedLenByteArray(
- reinterpret_cast<const uint8_t*>(&big_endian_values[j]) + offset);
- j += 2;
- }
- }
- }
-
- return WriteBatch<FLBAType>(num_levels, def_levels, rep_levels, buffer);
-}
-
-Status ArrowColumnWriter::Write(const Array& data) {
- ::arrow::Type::type values_type;
- RETURN_NOT_OK(GetLeafType(*data.type(), &values_type));
-
- std::shared_ptr<Array> _values_array;
- int64_t values_offset;
- int64_t num_levels;
- int64_t num_values;
- LevelBuilder level_builder(ctx_->memory_pool);
-
- std::shared_ptr<Buffer> def_levels_buffer, rep_levels_buffer;
- RETURN_NOT_OK(level_builder.GenerateLevels(
- data, field_, &values_offset, &num_values, &num_levels, ctx_->def_levels_buffer,
- &def_levels_buffer, &rep_levels_buffer, &_values_array));
- const int16_t* def_levels = nullptr;
- if (def_levels_buffer) {
- def_levels = reinterpret_cast<const int16_t*>(def_levels_buffer->data());
- }
- const int16_t* rep_levels = nullptr;
- if (rep_levels_buffer) {
- rep_levels = reinterpret_cast<const int16_t*>(rep_levels_buffer->data());
- }
- std::shared_ptr<Array> values_array = _values_array->Slice(values_offset, num_values);
-
-#define WRITE_BATCH_CASE(ArrowEnum, ArrowType, ParquetType) \
- case ::arrow::Type::ArrowEnum: \
- return TypedWriteBatch<ParquetType, ::arrow::ArrowType>(*values_array, num_levels, \
- def_levels, rep_levels);
-
- switch (values_type) {
- case ::arrow::Type::UINT32: {
- if (writer_->properties()->version() == ParquetVersion::PARQUET_1_0) {
- // Parquet 1.0 reader cannot read the UINT_32 logical type. Thus we need
- // to use the larger Int64Type to store them lossless.
- return TypedWriteBatch<Int64Type, ::arrow::UInt32Type>(*values_array, num_levels,
- def_levels, rep_levels);
- } else {
- return TypedWriteBatch<Int32Type, ::arrow::UInt32Type>(*values_array, num_levels,
- def_levels, rep_levels);
- }
- }
- WRITE_BATCH_CASE(NA, NullType, Int32Type)
- case ::arrow::Type::TIMESTAMP:
- return WriteTimestamps(*values_array, num_levels, def_levels, rep_levels);
- WRITE_BATCH_CASE(BOOL, BooleanType, BooleanType)
- WRITE_BATCH_CASE(INT8, Int8Type, Int32Type)
- WRITE_BATCH_CASE(UINT8, UInt8Type, Int32Type)
- WRITE_BATCH_CASE(INT16, Int16Type, Int32Type)
- WRITE_BATCH_CASE(UINT16, UInt16Type, Int32Type)
- WRITE_BATCH_CASE(INT32, Int32Type, Int32Type)
- WRITE_BATCH_CASE(INT64, Int64Type, Int64Type)
- WRITE_BATCH_CASE(UINT64, UInt64Type, Int64Type)
- WRITE_BATCH_CASE(FLOAT, FloatType, FloatType)
- WRITE_BATCH_CASE(DOUBLE, DoubleType, DoubleType)
- WRITE_BATCH_CASE(BINARY, BinaryType, ByteArrayType)
- WRITE_BATCH_CASE(STRING, BinaryType, ByteArrayType)
- WRITE_BATCH_CASE(FIXED_SIZE_BINARY, FixedSizeBinaryType, FLBAType)
- WRITE_BATCH_CASE(DECIMAL, Decimal128Type, FLBAType)
- WRITE_BATCH_CASE(DATE32, Date32Type, Int32Type)
- WRITE_BATCH_CASE(DATE64, Date64Type, Int32Type)
- WRITE_BATCH_CASE(TIME32, Time32Type, Int32Type)
- WRITE_BATCH_CASE(TIME64, Time64Type, Int64Type)
- default:
- break;
- }
- std::stringstream ss;
- ss << "Data type not supported as list value: " << values_array->type()->ToString();
- return Status::NotImplemented(ss.str());
-}
-
-} // namespace
-
-// ----------------------------------------------------------------------
-// FileWriter implementation
-
-class FileWriter::Impl {
- public:
- Impl(MemoryPool* pool, std::unique_ptr<ParquetFileWriter> writer,
- const std::shared_ptr<ArrowWriterProperties>& arrow_properties)
- : writer_(std::move(writer)),
- row_group_writer_(nullptr),
- column_write_context_(pool, arrow_properties.get()),
- arrow_properties_(arrow_properties),
- closed_(false) {}
-
- Status NewRowGroup(int64_t chunk_size) {
- if (row_group_writer_ != nullptr) {
- PARQUET_CATCH_NOT_OK(row_group_writer_->Close());
- }
- PARQUET_CATCH_NOT_OK(row_group_writer_ = writer_->AppendRowGroup());
- return Status::OK();
- }
-
- Status Close() {
- if (!closed_) {
- // Make idempotent
- closed_ = true;
- if (row_group_writer_ != nullptr) {
- PARQUET_CATCH_NOT_OK(row_group_writer_->Close());
- }
- PARQUET_CATCH_NOT_OK(writer_->Close());
- }
- return Status::OK();
- }
-
- Status WriteColumnChunk(const Array& data) {
- // A bit awkward here since cannot instantiate ChunkedArray from const Array&
- ::arrow::ArrayVector chunks = {::arrow::MakeArray(data.data())};
- auto chunked_array = std::make_shared<::arrow::ChunkedArray>(chunks);
- return WriteColumnChunk(chunked_array, 0, data.length());
- }
-
- Status WriteColumnChunk(const std::shared_ptr<ChunkedArray>& data, int64_t offset,
- const int64_t size) {
- // DictionaryArrays are not yet handled with a fast path. To still support
- // writing them as a workaround, we convert them back to their non-dictionary
- // representation.
- if (data->type()->id() == ::arrow::Type::DICTIONARY) {
- const ::arrow::DictionaryType& dict_type =
- static_cast<const ::arrow::DictionaryType&>(*data->type());
-
- // TODO(ARROW-1648): Remove this special handling once we require an Arrow
- // version that has this fixed.
- if (dict_type.dictionary()->type()->id() == ::arrow::Type::NA) {
- auto null_array = std::make_shared<::arrow::NullArray>(data->length());
- return WriteColumnChunk(*null_array);
- }
-
- FunctionContext ctx(this->memory_pool());
- ::arrow::compute::Datum cast_input(data);
- ::arrow::compute::Datum cast_output;
- RETURN_NOT_OK(Cast(&ctx, cast_input, dict_type.dictionary()->type(), CastOptions(),
- &cast_output));
- return WriteColumnChunk(cast_output.chunked_array(), offset, size);
- }
-
- ColumnWriter* column_writer;
- PARQUET_CATCH_NOT_OK(column_writer = row_group_writer_->NextColumn());
-
- // TODO(wesm): This trick to construct a schema for one Parquet root node
- // will not work for arbitrary nested data
- int current_column_idx = row_group_writer_->current_column();
- std::shared_ptr<::arrow::Schema> arrow_schema;
- RETURN_NOT_OK(FromParquetSchema(writer_->schema(), {current_column_idx - 1},
- writer_->key_value_metadata(), &arrow_schema));
-
- ArrowColumnWriter arrow_writer(&column_write_context_, column_writer,
- arrow_schema->field(0));
-
- RETURN_NOT_OK(arrow_writer.Write(*data, offset, size));
- return arrow_writer.Close();
- }
-
- const WriterProperties& properties() const { return *writer_->properties(); }
-
- ::arrow::MemoryPool* memory_pool() const { return column_write_context_.memory_pool; }
-
- virtual ~Impl() {}
-
- private:
- friend class FileWriter;
-
- std::unique_ptr<ParquetFileWriter> writer_;
- RowGroupWriter* row_group_writer_;
- ColumnWriterContext column_write_context_;
- std::shared_ptr<ArrowWriterProperties> arrow_properties_;
- bool closed_;
-};
-
-Status FileWriter::NewRowGroup(int64_t chunk_size) {
- return impl_->NewRowGroup(chunk_size);
-}
-
-Status FileWriter::WriteColumnChunk(const ::arrow::Array& data) {
- return impl_->WriteColumnChunk(data);
-}
-
-Status FileWriter::WriteColumnChunk(const std::shared_ptr<::arrow::ChunkedArray>& data,
- const int64_t offset, const int64_t size) {
- return impl_->WriteColumnChunk(data, offset, size);
-}
-
-Status FileWriter::WriteColumnChunk(const std::shared_ptr<::arrow::ChunkedArray>& data) {
- return WriteColumnChunk(data, 0, data->length());
-}
-
-Status FileWriter::Close() { return impl_->Close(); }
-
-MemoryPool* FileWriter::memory_pool() const { return impl_->memory_pool(); }
-
-FileWriter::~FileWriter() {}
-
-FileWriter::FileWriter(MemoryPool* pool, std::unique_ptr<ParquetFileWriter> writer,
- const std::shared_ptr<ArrowWriterProperties>& arrow_properties)
- : impl_(new FileWriter::Impl(pool, std::move(writer), arrow_properties)) {}
-
-Status FileWriter::Open(const ::arrow::Schema& schema, ::arrow::MemoryPool* pool,
- const std::shared_ptr<OutputStream>& sink,
- const std::shared_ptr<WriterProperties>& properties,
- std::unique_ptr<FileWriter>* writer) {
- return Open(schema, pool, sink, properties, default_arrow_writer_properties(), writer);
-}
-
-Status FileWriter::Open(const ::arrow::Schema& schema, ::arrow::MemoryPool* pool,
- const std::shared_ptr<OutputStream>& sink,
- const std::shared_ptr<WriterProperties>& properties,
- const std::shared_ptr<ArrowWriterProperties>& arrow_properties,
- std::unique_ptr<FileWriter>* writer) {
- std::shared_ptr<SchemaDescriptor> parquet_schema;
- RETURN_NOT_OK(
- ToParquetSchema(&schema, *properties, *arrow_properties, &parquet_schema));
-
- auto schema_node = std::static_pointer_cast<GroupNode>(parquet_schema->schema_root());
-
- std::unique_ptr<ParquetFileWriter> base_writer =
- ParquetFileWriter::Open(sink, schema_node, properties, schema.metadata());
-
- writer->reset(new FileWriter(pool, std::move(base_writer), arrow_properties));
- return Status::OK();
-}
-
-Status FileWriter::Open(const ::arrow::Schema& schema, ::arrow::MemoryPool* pool,
- const std::shared_ptr<::arrow::io::OutputStream>& sink,
- const std::shared_ptr<WriterProperties>& properties,
- std::unique_ptr<FileWriter>* writer) {
- auto wrapper = std::make_shared<ArrowOutputStream>(sink);
- return Open(schema, pool, wrapper, properties, writer);
-}
-
-Status FileWriter::Open(const ::arrow::Schema& schema, ::arrow::MemoryPool* pool,
- const std::shared_ptr<::arrow::io::OutputStream>& sink,
- const std::shared_ptr<WriterProperties>& properties,
- const std::shared_ptr<ArrowWriterProperties>& arrow_properties,
- std::unique_ptr<FileWriter>* writer) {
- auto wrapper = std::make_shared<ArrowOutputStream>(sink);
- return Open(schema, pool, wrapper, properties, arrow_properties, writer);
-}
-
-Status WriteFileMetaData(const FileMetaData& file_metadata, OutputStream* sink) {
- PARQUET_CATCH_NOT_OK(::parquet::WriteFileMetaData(file_metadata, sink));
- return Status::OK();
-}
-
-Status WriteFileMetaData(const FileMetaData& file_metadata,
- const std::shared_ptr<::arrow::io::OutputStream>& sink) {
- ArrowOutputStream wrapper(sink);
- return ::parquet::arrow::WriteFileMetaData(file_metadata, &wrapper);
-}
-
-namespace {} // namespace
-
-Status FileWriter::WriteTable(const Table& table, int64_t chunk_size) {
- if (chunk_size <= 0) {
- return Status::Invalid("chunk size per row_group must be greater than 0");
- } else if (chunk_size > impl_->properties().max_row_group_length()) {
- chunk_size = impl_->properties().max_row_group_length();
- }
-
- for (int chunk = 0; chunk * chunk_size < table.num_rows(); chunk++) {
- int64_t offset = chunk * chunk_size;
- int64_t size = std::min(chunk_size, table.num_rows() - offset);
-
- RETURN_NOT_OK_ELSE(NewRowGroup(size), PARQUET_IGNORE_NOT_OK(Close()));
- for (int i = 0; i < table.num_columns(); i++) {
- auto chunked_data = table.column(i)->data();
- RETURN_NOT_OK_ELSE(WriteColumnChunk(chunked_data, offset, size),
- PARQUET_IGNORE_NOT_OK(Close()));
- }
- }
- return Status::OK();
-}
-
-Status WriteTable(const ::arrow::Table& table, ::arrow::MemoryPool* pool,
- const std::shared_ptr<OutputStream>& sink, int64_t chunk_size,
- const std::shared_ptr<WriterProperties>& properties,
- const std::shared_ptr<ArrowWriterProperties>& arrow_properties) {
- std::unique_ptr<FileWriter> writer;
- RETURN_NOT_OK(FileWriter::Open(*table.schema(), pool, sink, properties,
- arrow_properties, &writer));
- RETURN_NOT_OK(writer->WriteTable(table, chunk_size));
- return writer->Close();
-}
-
-Status WriteTable(const ::arrow::Table& table, ::arrow::MemoryPool* pool,
- const std::shared_ptr<::arrow::io::OutputStream>& sink,
- int64_t chunk_size, const std::shared_ptr<WriterProperties>& properties,
- const std::shared_ptr<ArrowWriterProperties>& arrow_properties) {
- auto wrapper = std::make_shared<ArrowOutputStream>(sink);
- return WriteTable(table, pool, wrapper, chunk_size, properties, arrow_properties);
-}
-
-} // namespace arrow
-} // namespace parquet
diff --git a/src/parquet/arrow/writer.h b/src/parquet/arrow/writer.h
deleted file mode 100644
index ad6f1d5..0000000
--- a/src/parquet/arrow/writer.h
+++ /dev/null
@@ -1,210 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#ifndef PARQUET_ARROW_WRITER_H
-#define PARQUET_ARROW_WRITER_H
-
-#include <memory>
-
-#include "parquet/api/schema.h"
-#include "parquet/api/writer.h"
-
-#include "arrow/io/interfaces.h"
-#include "arrow/type.h"
-
-namespace arrow {
-
-class Array;
-class MemoryPool;
-class PrimitiveArray;
-class Schema;
-class Status;
-class StringArray;
-class Table;
-} // namespace arrow
-
-namespace parquet {
-namespace arrow {
-
-class PARQUET_EXPORT ArrowWriterProperties {
- public:
- class Builder {
- public:
- Builder() : write_nanos_as_int96_(false), coerce_timestamps_enabled_(false) {}
- virtual ~Builder() {}
-
- Builder* disable_deprecated_int96_timestamps() {
- write_nanos_as_int96_ = false;
- return this;
- }
-
- Builder* enable_deprecated_int96_timestamps() {
- write_nanos_as_int96_ = true;
- return this;
- }
-
- Builder* coerce_timestamps(::arrow::TimeUnit::type unit) {
- coerce_timestamps_enabled_ = true;
- coerce_timestamps_unit_ = unit;
- return this;
- }
-
- std::shared_ptr<ArrowWriterProperties> build() {
- return std::shared_ptr<ArrowWriterProperties>(new ArrowWriterProperties(
- write_nanos_as_int96_, coerce_timestamps_enabled_, coerce_timestamps_unit_));
- }
-
- private:
- bool write_nanos_as_int96_;
-
- bool coerce_timestamps_enabled_;
- ::arrow::TimeUnit::type coerce_timestamps_unit_;
- };
-
- bool support_deprecated_int96_timestamps() const { return write_nanos_as_int96_; }
-
- bool coerce_timestamps_enabled() const { return coerce_timestamps_enabled_; }
- ::arrow::TimeUnit::type coerce_timestamps_unit() const {
- return coerce_timestamps_unit_;
- }
-
- private:
- explicit ArrowWriterProperties(bool write_nanos_as_int96,
- bool coerce_timestamps_enabled,
- ::arrow::TimeUnit::type coerce_timestamps_unit)
- : write_nanos_as_int96_(write_nanos_as_int96),
- coerce_timestamps_enabled_(coerce_timestamps_enabled),
- coerce_timestamps_unit_(coerce_timestamps_unit) {}
-
- const bool write_nanos_as_int96_;
- const bool coerce_timestamps_enabled_;
- const ::arrow::TimeUnit::type coerce_timestamps_unit_;
-};
-
-std::shared_ptr<ArrowWriterProperties> PARQUET_EXPORT default_arrow_writer_properties();
-
-/**
- * Iterative API:
- * Start a new RowGroup/Chunk with NewRowGroup
- * Write column-by-column the whole column chunk
- */
-class PARQUET_EXPORT FileWriter {
- public:
- FileWriter(::arrow::MemoryPool* pool, std::unique_ptr<ParquetFileWriter> writer,
- const std::shared_ptr<ArrowWriterProperties>& arrow_properties =
- default_arrow_writer_properties());
-
- static ::arrow::Status Open(const ::arrow::Schema& schema, ::arrow::MemoryPool* pool,
- const std::shared_ptr<OutputStream>& sink,
- const std::shared_ptr<WriterProperties>& properties,
- std::unique_ptr<FileWriter>* writer);
-
- static ::arrow::Status Open(
- const ::arrow::Schema& schema, ::arrow::MemoryPool* pool,
- const std::shared_ptr<OutputStream>& sink,
- const std::shared_ptr<WriterProperties>& properties,
- const std::shared_ptr<ArrowWriterProperties>& arrow_properties,
- std::unique_ptr<FileWriter>* writer);
-
- static ::arrow::Status Open(const ::arrow::Schema& schema, ::arrow::MemoryPool* pool,
- const std::shared_ptr<::arrow::io::OutputStream>& sink,
- const std::shared_ptr<WriterProperties>& properties,
- std::unique_ptr<FileWriter>* writer);
-
- static ::arrow::Status Open(
- const ::arrow::Schema& schema, ::arrow::MemoryPool* pool,
- const std::shared_ptr<::arrow::io::OutputStream>& sink,
- const std::shared_ptr<WriterProperties>& properties,
- const std::shared_ptr<ArrowWriterProperties>& arrow_properties,
- std::unique_ptr<FileWriter>* writer);
-
- /// \brief Write a Table to Parquet.
- ::arrow::Status WriteTable(const ::arrow::Table& table, int64_t chunk_size);
-
- ::arrow::Status NewRowGroup(int64_t chunk_size);
- ::arrow::Status WriteColumnChunk(const ::arrow::Array& data);
-
- /// \brief Write ColumnChunk in row group using slice of a ChunkedArray
- ::arrow::Status WriteColumnChunk(const std::shared_ptr<::arrow::ChunkedArray>& data,
- const int64_t offset, const int64_t size);
- ::arrow::Status WriteColumnChunk(const std::shared_ptr<::arrow::ChunkedArray>& data);
- ::arrow::Status Close();
-
- virtual ~FileWriter();
-
- ::arrow::MemoryPool* memory_pool() const;
-
- private:
- class PARQUET_NO_EXPORT Impl;
- std::unique_ptr<Impl> impl_;
-};
-
-/// \brief Write Parquet file metadata only to indicated OutputStream
-PARQUET_EXPORT
-::arrow::Status WriteFileMetaData(const FileMetaData& file_metadata, OutputStream* sink);
-
-/// \brief Write Parquet file metadata only to indicated Arrow OutputStream
-PARQUET_EXPORT
-::arrow::Status WriteFileMetaData(const FileMetaData& file_metadata,
- const std::shared_ptr<::arrow::io::OutputStream>& sink);
-
-/**
- * Write a Table to Parquet.
- *
- * The table shall only consist of columns of primitive type or of primitive lists.
- */
-::arrow::Status PARQUET_EXPORT WriteTable(
- const ::arrow::Table& table, ::arrow::MemoryPool* pool,
- const std::shared_ptr<OutputStream>& sink, int64_t chunk_size,
- const std::shared_ptr<WriterProperties>& properties = default_writer_properties(),
- const std::shared_ptr<ArrowWriterProperties>& arrow_properties =
- default_arrow_writer_properties());
-
-::arrow::Status PARQUET_EXPORT WriteTable(
- const ::arrow::Table& table, ::arrow::MemoryPool* pool,
- const std::shared_ptr<::arrow::io::OutputStream>& sink, int64_t chunk_size,
- const std::shared_ptr<WriterProperties>& properties = default_writer_properties(),
- const std::shared_ptr<ArrowWriterProperties>& arrow_properties =
- default_arrow_writer_properties());
-
-namespace internal {
-
-/**
- * Timestamp conversion constants
- */
-constexpr int64_t kJulianEpochOffsetDays = INT64_C(2440588);
-constexpr int64_t kNanosecondsPerDay = INT64_C(86400000000000);
-
-/**
- * Converts nanosecond timestamps to Impala (Int96) format
- */
-inline void NanosecondsToImpalaTimestamp(const int64_t nanoseconds,
- Int96* impala_timestamp) {
- int64_t julian_days = (nanoseconds / kNanosecondsPerDay) + kJulianEpochOffsetDays;
- (*impala_timestamp).value[2] = (uint32_t)julian_days;
-
- int64_t last_day_nanos = nanoseconds % kNanosecondsPerDay;
- int64_t* impala_last_day_nanos = reinterpret_cast<int64_t*>(impala_timestamp);
- *impala_last_day_nanos = last_day_nanos;
-}
-
-} // namespace internal
-
-} // namespace arrow
-} // namespace parquet
-
-#endif // PARQUET_ARROW_WRITER_H
diff --git a/src/parquet/bloom_filter-test.cc b/src/parquet/bloom_filter-test.cc
deleted file mode 100644
index 96d2e06..0000000
--- a/src/parquet/bloom_filter-test.cc
+++ /dev/null
@@ -1,244 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#include <gtest/gtest.h>
-
-#include <algorithm>
-#include <random>
-#include <string>
-
-#include "arrow/io/file.h"
-#include "parquet/bloom_filter.h"
-#include "parquet/murmur3.h"
-#include "parquet/util/memory.h"
-#include "parquet/util/test-common.h"
-
-namespace parquet {
-namespace test {
-TEST(Murmur3Test, TestBloomFilter) {
- uint64_t result;
- const uint8_t bitset[8] = {0x0, 0x1, 0x2, 0x3, 0x4, 0x5, 0x6, 0x7};
- ByteArray byteArray(8, bitset);
- MurmurHash3 murmur3;
- result = murmur3.Hash(&byteArray);
- EXPECT_EQ(result, UINT64_C(913737700387071329));
-}
-
-TEST(ConstructorTest, TestBloomFilter) {
- BlockSplitBloomFilter bloom_filter;
- EXPECT_NO_THROW(bloom_filter.Init(1000));
-
- // It throws because the length cannot be zero
- std::unique_ptr<uint8_t[]> bitset1(new uint8_t[1024]());
- EXPECT_THROW(bloom_filter.Init(bitset1.get(), 0), ParquetException);
-
- // It throws because the number of bytes of Bloom filter bitset must be a power of 2.
- std::unique_ptr<uint8_t[]> bitset2(new uint8_t[1024]());
- EXPECT_THROW(bloom_filter.Init(bitset2.get(), 1023), ParquetException);
-}
-
-// The BasicTest is used to test basic operations including InsertHash, FindHash and
-// serializing and de-serializing.
-TEST(BasicTest, TestBloomFilter) {
- BlockSplitBloomFilter bloom_filter;
- bloom_filter.Init(1024);
-
- for (int i = 0; i < 10; i++) {
- bloom_filter.InsertHash(bloom_filter.Hash(i));
- }
-
- for (int i = 0; i < 10; i++) {
- EXPECT_TRUE(bloom_filter.FindHash(bloom_filter.Hash(i)));
- }
-
- // Serialize Bloom filter to memory output stream
- InMemoryOutputStream sink;
- bloom_filter.WriteTo(&sink);
-
- // Deserialize Bloom filter from memory
- InMemoryInputStream source(sink.GetBuffer());
-
- BlockSplitBloomFilter de_bloom = BlockSplitBloomFilter::Deserialize(&source);
-
- for (int i = 0; i < 10; i++) {
- EXPECT_TRUE(de_bloom.FindHash(de_bloom.Hash(i)));
- }
-}
-
-// Helper function to generate random string.
-std::string GetRandomString(uint32_t length) {
- // Character set used to generate random string
- const std::string charset =
- "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz";
-
- // The uuid_seed was generated by "uuidgen -r"
- const std::string uuid_seed = "8de406aa-fb59-4195-a81c-5152af26433f";
- std::seed_seq seed(uuid_seed.begin(), uuid_seed.end());
- std::mt19937 generator(seed);
- std::uniform_int_distribution<uint32_t> dist(0, static_cast<int>(charset.size() - 1));
- std::string ret = "";
-
- for (uint32_t i = 0; i < length; i++) {
- ret += charset[dist(generator)];
- }
-
- return ret;
-}
-
-TEST(FPPTest, TestBloomFilter) {
- // It counts the number of times FindHash returns true.
- int exist = 0;
-
- // Total count of elements that will be used
-#ifdef PARQUET_VALGRIND
- const int total_count = 5000;
-#else
- const int total_count = 100000;
-#endif
-
- // Bloom filter fpp parameter
- const double fpp = 0.01;
-
- std::vector<std::string> members;
- BlockSplitBloomFilter bloom_filter;
- bloom_filter.Init(BlockSplitBloomFilter::OptimalNumOfBits(total_count, fpp));
-
- // Insert elements into the Bloom filter
- for (int i = 0; i < total_count; i++) {
- // Insert random string which length is 8
- std::string tmp = GetRandomString(8);
- const ByteArray byte_array(8, reinterpret_cast<const uint8_t*>(tmp.c_str()));
- members.push_back(tmp);
- bloom_filter.InsertHash(bloom_filter.Hash(&byte_array));
- }
-
- for (int i = 0; i < total_count; i++) {
- const ByteArray byte_array1(8, reinterpret_cast<const uint8_t*>(members[i].c_str()));
- ASSERT_TRUE(bloom_filter.FindHash(bloom_filter.Hash(&byte_array1)));
- std::string tmp = GetRandomString(7);
- const ByteArray byte_array2(7, reinterpret_cast<const uint8_t*>(tmp.c_str()));
-
- if (bloom_filter.FindHash(bloom_filter.Hash(&byte_array2))) {
- exist++;
- }
- }
-
- // The exist should be probably less than 1000 according default FPP 0.01.
- EXPECT_TRUE(exist < total_count * fpp);
-}
-
-// The CompatibilityTest is used to test cross compatibility with parquet-mr, it reads
-// the Bloom filter binary generated by the Bloom filter class in the parquet-mr project
-// and tests whether the values inserted before could be filtered or not.
-
-// The Bloom filter binary is generated by three steps in from Parquet-mr.
-// Step 1: Construct a Bloom filter with 1024 bytes bitset.
-// Step 2: Insert "hello", "parquet", "bloom", "filter" to Bloom filter.
-// Step 3: Call writeTo API to write to File.
-TEST(CompatibilityTest, TestBloomFilter) {
- const std::string test_string[4] = {"hello", "parquet", "bloom", "filter"};
- const std::string bloom_filter_test_binary =
- std::string(test::get_data_dir()) + "/bloom_filter.bin";
- std::shared_ptr<::arrow::io::ReadableFile> handle;
-
- int64_t size;
- PARQUET_THROW_NOT_OK(
- ::arrow::io::ReadableFile::Open(bloom_filter_test_binary, &handle));
- PARQUET_THROW_NOT_OK(handle->GetSize(&size));
-
- // 1024 bytes (bitset) + 4 bytes (hash) + 4 bytes (algorithm) + 4 bytes (length)
- EXPECT_EQ(size, 1036);
-
- std::unique_ptr<uint8_t[]> bitset(new uint8_t[size]());
- std::shared_ptr<Buffer> buffer(new Buffer(bitset.get(), size));
- PARQUET_THROW_NOT_OK(handle->Read(size, &buffer));
-
- InMemoryInputStream source(buffer);
- BlockSplitBloomFilter bloom_filter1 = BlockSplitBloomFilter::Deserialize(&source);
-
- for (int i = 0; i < 4; i++) {
- const ByteArray tmp(static_cast<uint32_t>(test_string[i].length()),
- reinterpret_cast<const uint8_t*>(test_string[i].c_str()));
- EXPECT_TRUE(bloom_filter1.FindHash(bloom_filter1.Hash(&tmp)));
- }
-
- // The following is used to check whether the new created Bloom filter in parquet-cpp is
- // byte-for-byte identical to file at bloom_data_path which is created from parquet-mr
- // with same inserted hashes.
- BlockSplitBloomFilter bloom_filter2;
- bloom_filter2.Init(bloom_filter1.GetBitsetSize());
- for (int i = 0; i < 4; i++) {
- const ByteArray byte_array(static_cast<uint32_t>(test_string[i].length()),
- reinterpret_cast<const uint8_t*>(test_string[i].c_str()));
- bloom_filter2.InsertHash(bloom_filter2.Hash(&byte_array));
- }
-
- // Serialize Bloom filter to memory output stream
- InMemoryOutputStream sink;
- bloom_filter2.WriteTo(&sink);
- std::shared_ptr<Buffer> buffer1 = sink.GetBuffer();
-
- PARQUET_THROW_NOT_OK(handle->Seek(0));
- PARQUET_THROW_NOT_OK(handle->GetSize(&size));
- std::shared_ptr<Buffer> buffer2;
- PARQUET_THROW_NOT_OK(handle->Read(size, &buffer2));
-
- EXPECT_TRUE((*buffer1).Equals(*buffer2));
-}
-
-// OptmialValueTest is used to test whether OptimalNumOfBits returns expected
-// numbers according to formula:
-// num_of_bits = -8.0 * ndv / log(1 - pow(fpp, 1.0 / 8.0))
-// where ndv is the number of distinct values and fpp is the false positive probability.
-// Also it is used to test whether OptimalNumOfBits returns value between
-// [MINIMUM_BLOOM_FILTER_SIZE, MAXIMUM_BLOOM_FILTER_SIZE].
-TEST(OptimalValueTest, TestBloomFilter) {
- EXPECT_EQ(BlockSplitBloomFilter::OptimalNumOfBits(256, 0.01), UINT32_C(4096));
- EXPECT_EQ(BlockSplitBloomFilter::OptimalNumOfBits(512, 0.01), UINT32_C(8192));
- EXPECT_EQ(BlockSplitBloomFilter::OptimalNumOfBits(1024, 0.01), UINT32_C(16384));
- EXPECT_EQ(BlockSplitBloomFilter::OptimalNumOfBits(2048, 0.01), UINT32_C(32768));
-
- EXPECT_EQ(BlockSplitBloomFilter::OptimalNumOfBits(200, 0.01), UINT32_C(2048));
- EXPECT_EQ(BlockSplitBloomFilter::OptimalNumOfBits(300, 0.01), UINT32_C(4096));
- EXPECT_EQ(BlockSplitBloomFilter::OptimalNumOfBits(700, 0.01), UINT32_C(8192));
- EXPECT_EQ(BlockSplitBloomFilter::OptimalNumOfBits(1500, 0.01), UINT32_C(16384));
-
- EXPECT_EQ(BlockSplitBloomFilter::OptimalNumOfBits(200, 0.025), UINT32_C(2048));
- EXPECT_EQ(BlockSplitBloomFilter::OptimalNumOfBits(300, 0.025), UINT32_C(4096));
- EXPECT_EQ(BlockSplitBloomFilter::OptimalNumOfBits(700, 0.025), UINT32_C(8192));
- EXPECT_EQ(BlockSplitBloomFilter::OptimalNumOfBits(1500, 0.025), UINT32_C(16384));
-
- EXPECT_EQ(BlockSplitBloomFilter::OptimalNumOfBits(200, 0.05), UINT32_C(2048));
- EXPECT_EQ(BlockSplitBloomFilter::OptimalNumOfBits(300, 0.05), UINT32_C(4096));
- EXPECT_EQ(BlockSplitBloomFilter::OptimalNumOfBits(700, 0.05), UINT32_C(8192));
- EXPECT_EQ(BlockSplitBloomFilter::OptimalNumOfBits(1500, 0.05), UINT32_C(16384));
-
- // Boundary check
- EXPECT_EQ(BlockSplitBloomFilter::OptimalNumOfBits(4, 0.01), UINT32_C(256));
- EXPECT_EQ(BlockSplitBloomFilter::OptimalNumOfBits(4, 0.25), UINT32_C(256));
-
- EXPECT_EQ(
- BlockSplitBloomFilter::OptimalNumOfBits(std::numeric_limits<uint32_t>::max(), 0.01),
- UINT32_C(1073741824));
- EXPECT_EQ(
- BlockSplitBloomFilter::OptimalNumOfBits(std::numeric_limits<uint32_t>::max(), 0.25),
- UINT32_C(1073741824));
-}
-
-} // namespace test
-
-} // namespace parquet
diff --git a/src/parquet/bloom_filter.cc b/src/parquet/bloom_filter.cc
deleted file mode 100644
index faa344c..0000000
--- a/src/parquet/bloom_filter.cc
+++ /dev/null
@@ -1,166 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#include <algorithm>
-#include <cmath>
-#include <cstdint>
-
-#include "arrow/status.h"
-#include "arrow/util/bit-util.h"
-#include "parquet/bloom_filter.h"
-#include "parquet/exception.h"
-#include "parquet/murmur3.h"
-#include "parquet/types.h"
-#include "parquet/util/logging.h"
-
-namespace parquet {
-constexpr uint32_t BlockSplitBloomFilter::SALT[kBitsSetPerBlock];
-
-BlockSplitBloomFilter::BlockSplitBloomFilter()
- : pool_(::arrow::default_memory_pool()),
- hash_strategy_(HashStrategy::MURMUR3_X64_128),
- algorithm_(Algorithm::BLOCK) {}
-
-void BlockSplitBloomFilter::Init(uint32_t num_bytes) {
- if (num_bytes < kMinimumBloomFilterBytes) {
- num_bytes = kMinimumBloomFilterBytes;
- }
-
- // Get next power of 2 if it is not power of 2.
- if ((num_bytes & (num_bytes - 1)) != 0) {
- num_bytes = static_cast<uint32_t>(::arrow::BitUtil::NextPower2(num_bytes));
- }
-
- if (num_bytes > kMaximumBloomFilterBytes) {
- num_bytes = kMaximumBloomFilterBytes;
- }
-
- num_bytes_ = num_bytes;
- PARQUET_THROW_NOT_OK(::arrow::AllocateBuffer(pool_, num_bytes_, &data_));
- memset(data_->mutable_data(), 0, num_bytes_);
-
- this->hasher_.reset(new MurmurHash3());
-}
-
-void BlockSplitBloomFilter::Init(const uint8_t* bitset, uint32_t num_bytes) {
- DCHECK(bitset != nullptr);
-
- if (num_bytes < kMinimumBloomFilterBytes || num_bytes > kMaximumBloomFilterBytes ||
- (num_bytes & (num_bytes - 1)) != 0) {
- throw ParquetException("Given length of bitset is illegal");
- }
-
- num_bytes_ = num_bytes;
- PARQUET_THROW_NOT_OK(::arrow::AllocateBuffer(pool_, num_bytes_, &data_));
- memcpy(data_->mutable_data(), bitset, num_bytes_);
-
- this->hasher_.reset(new MurmurHash3());
-}
-
-BlockSplitBloomFilter BlockSplitBloomFilter::Deserialize(InputStream* input) {
- int64_t bytes_available;
-
- const uint8_t* read_buffer = NULL;
- read_buffer = input->Read(sizeof(uint32_t), &bytes_available);
- if (static_cast<uint32_t>(bytes_available) != sizeof(uint32_t) || !read_buffer) {
- throw ParquetException("Failed to deserialize from input stream");
- }
- uint32_t len;
- memcpy(&len, read_buffer, sizeof(uint32_t));
-
- read_buffer = input->Read(sizeof(uint32_t), &bytes_available);
- if (static_cast<uint32_t>(bytes_available) != sizeof(uint32_t) || !read_buffer) {
- throw ParquetException("Failed to deserialize from input stream");
- }
- uint32_t hash;
- memcpy(&hash, read_buffer, sizeof(uint32_t));
- if (static_cast<HashStrategy>(hash) != HashStrategy::MURMUR3_X64_128) {
- throw ParquetException("Unsupported hash strategy");
- }
-
- read_buffer = input->Read(sizeof(uint32_t), &bytes_available);
- if (static_cast<uint32_t>(bytes_available) != sizeof(uint32_t) || !read_buffer) {
- throw ParquetException("Failed to deserialize from input stream");
- }
- uint32_t algorithm;
- memcpy(&algorithm, read_buffer, sizeof(uint32_t));
- if (static_cast<Algorithm>(algorithm) != BloomFilter::Algorithm::BLOCK) {
- throw ParquetException("Unsupported Bloom filter algorithm");
- }
-
- BlockSplitBloomFilter bloom_filter;
- bloom_filter.Init(input->Read(len, &bytes_available), len);
- return bloom_filter;
-}
-
-void BlockSplitBloomFilter::WriteTo(OutputStream* sink) const {
- DCHECK(sink != nullptr);
-
- sink->Write(reinterpret_cast<const uint8_t*>(&num_bytes_), sizeof(num_bytes_));
- sink->Write(reinterpret_cast<const uint8_t*>(&hash_strategy_), sizeof(hash_strategy_));
- sink->Write(reinterpret_cast<const uint8_t*>(&algorithm_), sizeof(algorithm_));
- sink->Write(data_->mutable_data(), num_bytes_);
-}
-
-void BlockSplitBloomFilter::SetMask(uint32_t key, BlockMask& block_mask) const {
- for (int i = 0; i < kBitsSetPerBlock; ++i) {
- block_mask.item[i] = key * SALT[i];
- }
-
- for (int i = 0; i < kBitsSetPerBlock; ++i) {
- block_mask.item[i] = block_mask.item[i] >> 27;
- }
-
- for (int i = 0; i < kBitsSetPerBlock; ++i) {
- block_mask.item[i] = UINT32_C(0x1) << block_mask.item[i];
- }
-}
-
-bool BlockSplitBloomFilter::FindHash(uint64_t hash) const {
- const uint32_t bucket_index =
- static_cast<uint32_t>((hash >> 32) & (num_bytes_ / kBytesPerFilterBlock - 1));
- uint32_t key = static_cast<uint32_t>(hash);
- uint32_t* bitset32 = reinterpret_cast<uint32_t*>(data_->mutable_data());
-
- // Calculate mask for bucket.
- BlockMask block_mask;
- SetMask(key, block_mask);
-
- for (int i = 0; i < kBitsSetPerBlock; ++i) {
- if (0 == (bitset32[kBitsSetPerBlock * bucket_index + i] & block_mask.item[i])) {
- return false;
- }
- }
- return true;
-}
-
-void BlockSplitBloomFilter::InsertHash(uint64_t hash) {
- const uint32_t bucket_index =
- static_cast<uint32_t>(hash >> 32) & (num_bytes_ / kBytesPerFilterBlock - 1);
- uint32_t key = static_cast<uint32_t>(hash);
- uint32_t* bitset32 = reinterpret_cast<uint32_t*>(data_->mutable_data());
-
- // Calculate mask for bucket.
- BlockMask block_mask;
- SetMask(key, block_mask);
-
- for (int i = 0; i < kBitsSetPerBlock; i++) {
- bitset32[bucket_index * kBitsSetPerBlock + i] |= block_mask.item[i];
- }
-}
-
-} // namespace parquet
diff --git a/src/parquet/bloom_filter.h b/src/parquet/bloom_filter.h
deleted file mode 100644
index e39370a..0000000
--- a/src/parquet/bloom_filter.h
+++ /dev/null
@@ -1,244 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#ifndef PARQUET_BLOOM_FILTER_H
-#define PARQUET_BLOOM_FILTER_H
-
-#include <cstdint>
-
-#include "parquet/exception.h"
-#include "parquet/hasher.h"
-#include "parquet/types.h"
-#include "parquet/util/logging.h"
-#include "parquet/util/memory.h"
-
-namespace parquet {
-class OutputStream;
-
-// A Bloom filter is a compact structure to indicate whether an item is not in a set or
-// probably in a set. The Bloom filter usually consists of a bit set that represents a
-// set of elements, a hash strategy and a Bloom filter algorithm.
-class BloomFilter {
- public:
- // Maximum Bloom filter size, it sets to HDFS default block size 128MB
- // This value will be reconsidered when implementing Bloom filter producer.
- static constexpr uint32_t kMaximumBloomFilterBytes = 128 * 1024 * 1024;
-
- /// Determine whether an element exist in set or not.
- ///
- /// @param hash the element to contain.
- /// @return false if value is definitely not in set, and true means PROBABLY
- /// in set.
- virtual bool FindHash(uint64_t hash) const = 0;
-
- /// Insert element to set represented by Bloom filter bitset.
- /// @param hash the hash of value to insert into Bloom filter.
- virtual void InsertHash(uint64_t hash) = 0;
-
- /// Write this Bloom filter to an output stream. A Bloom filter structure should
- /// include bitset length, hash strategy, algorithm, and bitset.
- ///
- /// @param sink the output stream to write
- virtual void WriteTo(OutputStream* sink) const = 0;
-
- /// Get the number of bytes of bitset
- virtual uint32_t GetBitsetSize() const = 0;
-
- /// Compute hash for 32 bits value by using its plain encoding result.
- ///
- /// @param value the value to hash.
- /// @return hash result.
- virtual uint64_t Hash(int32_t value) const = 0;
-
- /// Compute hash for 64 bits value by using its plain encoding result.
- ///
- /// @param value the value to hash.
- /// @return hash result.
- virtual uint64_t Hash(int64_t value) const = 0;
-
- /// Compute hash for float value by using its plain encoding result.
- ///
- /// @param value the value to hash.
- /// @return hash result.
- virtual uint64_t Hash(float value) const = 0;
-
- /// Compute hash for double value by using its plain encoding result.
- ///
- /// @param value the value to hash.
- /// @return hash result.
- virtual uint64_t Hash(double value) const = 0;
-
- /// Compute hash for Int96 value by using its plain encoding result.
- ///
- /// @param value the value to hash.
- /// @return hash result.
- virtual uint64_t Hash(const Int96* value) const = 0;
-
- /// Compute hash for ByteArray value by using its plain encoding result.
- ///
- /// @param value the value to hash.
- /// @return hash result.
- virtual uint64_t Hash(const ByteArray* value) const = 0;
-
- /// Compute hash for fixed byte array value by using its plain encoding result.
- ///
- /// @param value the value to hash.
- /// @return hash result.
- virtual uint64_t Hash(const FLBA* value, uint32_t len) const = 0;
-
- virtual ~BloomFilter() {}
-
- protected:
- // Hash strategy available for Bloom filter.
- enum class HashStrategy : uint32_t { MURMUR3_X64_128 = 0 };
-
- // Bloom filter algorithm.
- enum class Algorithm : uint32_t { BLOCK = 0 };
-};
-
-// The BlockSplitBloomFilter is implemented using block-based Bloom filters from
-// Putze et al.'s "Cache-,Hash- and Space-Efficient Bloom filters". The basic idea is to
-// hash the item to a tiny Bloom filter which size fit a single cache line or smaller.
-//
-// This implementation sets 8 bits in each tiny Bloom filter. Each tiny Bloom
-// filter is 32 bytes to take advantage of 32-byte SIMD instructions.
-class BlockSplitBloomFilter : public BloomFilter {
- public:
- /// The constructor of BlockSplitBloomFilter. It uses murmur3_x64_128 as hash function.
- BlockSplitBloomFilter();
-
- /// Initialize the BlockSplitBloomFilter. The range of num_bytes should be within
- /// [kMinimumBloomFilterBytes, kMaximumBloomFilterBytes], it will be
- /// rounded up/down to lower/upper bound if num_bytes is out of range and also
- /// will be rounded up to a power of 2.
- ///
- /// @param num_bytes The number of bytes to store Bloom filter bitset.
- void Init(uint32_t num_bytes);
-
- /// Initialize the BlockSplitBloomFilter. It copies the bitset as underlying
- /// bitset because the given bitset may not satisfy the 32-byte alignment requirement
- /// which may lead to segfault when performing SIMD instructions. It is the caller's
- /// responsibility to free the bitset passed in. This is used when reconstructing
- /// a Bloom filter from a parquet file.
- ///
- /// @param bitset The given bitset to initialize the Bloom filter.
- /// @param num_bytes The number of bytes of given bitset.
- void Init(const uint8_t* bitset, uint32_t num_bytes);
-
- // Minimum Bloom filter size, it sets to 32 bytes to fit a tiny Bloom filter.
- static constexpr uint32_t kMinimumBloomFilterBytes = 32;
-
- /// Calculate optimal size according to the number of distinct values and false
- /// positive probability.
- ///
- /// @param ndv The number of distinct values.
- /// @param fpp The false positive probability.
- /// @return it always return a value between kMinimumBloomFilterBytes and
- /// kMaximumBloomFilterBytes, and the return value is always a power of 2
- static uint32_t OptimalNumOfBits(uint32_t ndv, double fpp) {
- DCHECK(fpp > 0.0 && fpp < 1.0);
- const double m = -8.0 * ndv / log(1 - pow(fpp, 1.0 / 8));
- uint32_t num_bits = static_cast<uint32_t>(m);
-
- // Handle overflow.
- if (m < 0 || m > kMaximumBloomFilterBytes << 3) {
- num_bits = static_cast<uint32_t>(kMaximumBloomFilterBytes << 3);
- }
-
- // Round up to lower bound
- if (num_bits < kMinimumBloomFilterBytes << 3) {
- num_bits = kMinimumBloomFilterBytes << 3;
- }
-
- // Get next power of 2 if bits is not power of 2.
- if ((num_bits & (num_bits - 1)) != 0) {
- num_bits = static_cast<uint32_t>(::arrow::BitUtil::NextPower2(num_bits));
- }
-
- // Round down to upper bound
- if (num_bits > kMaximumBloomFilterBytes << 3) {
- num_bits = kMaximumBloomFilterBytes << 3;
- }
-
- return num_bits;
- }
-
- bool FindHash(uint64_t hash) const override;
- void InsertHash(uint64_t hash) override;
- void WriteTo(OutputStream* sink) const override;
- uint32_t GetBitsetSize() const override { return num_bytes_; }
- uint64_t Hash(int64_t value) const override { return hasher_->Hash(value); }
- uint64_t Hash(float value) const override { return hasher_->Hash(value); }
- uint64_t Hash(double value) const override { return hasher_->Hash(value); }
- uint64_t Hash(const Int96* value) const override { return hasher_->Hash(value); }
- uint64_t Hash(const ByteArray* value) const override { return hasher_->Hash(value); }
- uint64_t Hash(int32_t value) const override { return hasher_->Hash(value); }
- uint64_t Hash(const FLBA* value, uint32_t len) const override {
- return hasher_->Hash(value, len);
- }
- /// Deserialize the Bloom filter from an input stream. It is used when reconstructing
- /// a Bloom filter from a parquet filter.
- ///
- /// @param input_stream The input stream from which to construct the Bloom filter
- /// @return The BlockSplitBloomFilter.
- static BlockSplitBloomFilter Deserialize(InputStream* input_stream);
-
- private:
- // Bytes in a tiny Bloom filter block.
- static constexpr int kBytesPerFilterBlock = 32;
-
- // The number of bits to be set in each tiny Bloom filter
- static constexpr int kBitsSetPerBlock = 8;
-
- // A mask structure used to set bits in each tiny Bloom filter.
- struct BlockMask {
- uint32_t item[kBitsSetPerBlock];
- };
-
- // The block-based algorithm needs eight odd SALT values to calculate eight indexes
- // of bit to set, one bit in each 32-bit word.
- static constexpr uint32_t SALT[kBitsSetPerBlock] = {
- 0x47b6137bU, 0x44974d91U, 0x8824ad5bU, 0xa2b7289dU,
- 0x705495c7U, 0x2df1424bU, 0x9efc4947U, 0x5c6bfb31U};
-
- /// Set bits in mask array according to input key.
- /// @param key the value to calculate mask values.
- /// @param mask the mask array is used to set inside a block
- void SetMask(uint32_t key, BlockMask& mask) const;
-
- // Memory pool to allocate aligned buffer for bitset
- ::arrow::MemoryPool* pool_;
-
- // The underlying buffer of bitset.
- std::shared_ptr<Buffer> data_;
-
- // The number of bytes of Bloom filter bitset.
- uint32_t num_bytes_;
-
- // Hash strategy used in this Bloom filter.
- HashStrategy hash_strategy_;
-
- // Algorithm used in this Bloom filter.
- Algorithm algorithm_;
-
- // The hash pointer points to actual hash class used.
- std::unique_ptr<Hasher> hasher_;
-};
-
-} // namespace parquet
-
-#endif // PARQUET_BLOOM_FILTER_H
diff --git a/src/parquet/column-io-benchmark.cc b/src/parquet/column-io-benchmark.cc
deleted file mode 100644
index a9a7530..0000000
--- a/src/parquet/column-io-benchmark.cc
+++ /dev/null
@@ -1,237 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#include "benchmark/benchmark.h"
-
-#include "parquet/column_reader.h"
-#include "parquet/column_writer.h"
-#include "parquet/file_reader.h"
-#include "parquet/parquet_types.h"
-#include "parquet/util/memory.h"
-
-namespace parquet {
-
-using schema::PrimitiveNode;
-
-namespace benchmark {
-
-std::unique_ptr<Int64Writer> BuildWriter(int64_t output_size, OutputStream* dst,
- ColumnChunkMetaDataBuilder* metadata,
- ColumnDescriptor* schema,
- const WriterProperties* properties) {
- std::unique_ptr<PageWriter> pager =
- PageWriter::Open(dst, Compression::UNCOMPRESSED, metadata);
- return std::unique_ptr<Int64Writer>(
- new Int64Writer(metadata, std::move(pager), Encoding::PLAIN, properties));
-}
-
-std::shared_ptr<ColumnDescriptor> Int64Schema(Repetition::type repetition) {
- auto node = PrimitiveNode::Make("int64", repetition, Type::INT64);
- return std::make_shared<ColumnDescriptor>(node, repetition != Repetition::REQUIRED,
- repetition == Repetition::REPEATED);
-}
-
-void SetBytesProcessed(::benchmark::State& state, Repetition::type repetition) {
- int64_t bytes_processed = state.iterations() * state.range(0) * sizeof(int64_t);
- if (repetition != Repetition::REQUIRED) {
- bytes_processed += state.iterations() * state.range(0) * sizeof(int16_t);
- }
- if (repetition == Repetition::REPEATED) {
- bytes_processed += state.iterations() * state.range(0) * sizeof(int16_t);
- }
- state.SetBytesProcessed(state.iterations() * state.range(0) * sizeof(int16_t));
-}
-
-template <Repetition::type repetition,
- Compression::type codec = Compression::UNCOMPRESSED>
-static void BM_WriteInt64Column(::benchmark::State& state) {
- format::ColumnChunk thrift_metadata;
- std::vector<int64_t> values(state.range(0), 128);
- std::vector<int16_t> definition_levels(state.range(0), 1);
- std::vector<int16_t> repetition_levels(state.range(0), 0);
- std::shared_ptr<ColumnDescriptor> schema = Int64Schema(repetition);
- WriterProperties::Builder builder;
- std::shared_ptr<WriterProperties> properties = builder.compression(codec)->build();
- auto metadata = ColumnChunkMetaDataBuilder::Make(
- properties, schema.get(), reinterpret_cast<uint8_t*>(&thrift_metadata));
-
- while (state.KeepRunning()) {
- InMemoryOutputStream stream;
- std::unique_ptr<Int64Writer> writer = BuildWriter(
- state.range(0), &stream, metadata.get(), schema.get(), properties.get());
- writer->WriteBatch(values.size(), definition_levels.data(), repetition_levels.data(),
- values.data());
- writer->Close();
- }
- SetBytesProcessed(state, repetition);
-}
-
-BENCHMARK_TEMPLATE(BM_WriteInt64Column, Repetition::REQUIRED)->Range(1024, 65536);
-
-BENCHMARK_TEMPLATE(BM_WriteInt64Column, Repetition::OPTIONAL)->Range(1024, 65536);
-
-BENCHMARK_TEMPLATE(BM_WriteInt64Column, Repetition::REPEATED)->Range(1024, 65536);
-
-BENCHMARK_TEMPLATE(BM_WriteInt64Column, Repetition::REQUIRED, Compression::SNAPPY)
- ->Range(1024, 65536);
-BENCHMARK_TEMPLATE(BM_WriteInt64Column, Repetition::OPTIONAL, Compression::SNAPPY)
- ->Range(1024, 65536);
-BENCHMARK_TEMPLATE(BM_WriteInt64Column, Repetition::REPEATED, Compression::SNAPPY)
- ->Range(1024, 65536);
-
-BENCHMARK_TEMPLATE(BM_WriteInt64Column, Repetition::REQUIRED, Compression::LZ4)
- ->Range(1024, 65536);
-BENCHMARK_TEMPLATE(BM_WriteInt64Column, Repetition::OPTIONAL, Compression::LZ4)
- ->Range(1024, 65536);
-BENCHMARK_TEMPLATE(BM_WriteInt64Column, Repetition::REPEATED, Compression::LZ4)
- ->Range(1024, 65536);
-
-BENCHMARK_TEMPLATE(BM_WriteInt64Column, Repetition::REQUIRED, Compression::ZSTD)
- ->Range(1024, 65536);
-BENCHMARK_TEMPLATE(BM_WriteInt64Column, Repetition::OPTIONAL, Compression::ZSTD)
- ->Range(1024, 65536);
-BENCHMARK_TEMPLATE(BM_WriteInt64Column, Repetition::REPEATED, Compression::ZSTD)
- ->Range(1024, 65536);
-
-std::unique_ptr<Int64Reader> BuildReader(std::shared_ptr<Buffer>& buffer,
- int64_t num_values, ColumnDescriptor* schema) {
- std::unique_ptr<InMemoryInputStream> source(new InMemoryInputStream(buffer));
- std::unique_ptr<PageReader> page_reader =
- PageReader::Open(std::move(source), num_values, Compression::UNCOMPRESSED);
- return std::unique_ptr<Int64Reader>(new Int64Reader(schema, std::move(page_reader)));
-}
-
-template <Repetition::type repetition,
- Compression::type codec = Compression::UNCOMPRESSED>
-static void BM_ReadInt64Column(::benchmark::State& state) {
- format::ColumnChunk thrift_metadata;
- std::vector<int64_t> values(state.range(0), 128);
- std::vector<int16_t> definition_levels(state.range(0), 1);
- std::vector<int16_t> repetition_levels(state.range(0), 0);
- std::shared_ptr<ColumnDescriptor> schema = Int64Schema(repetition);
- WriterProperties::Builder builder;
- std::shared_ptr<WriterProperties> properties = builder.compression(codec)->build();
- auto metadata = ColumnChunkMetaDataBuilder::Make(
- properties, schema.get(), reinterpret_cast<uint8_t*>(&thrift_metadata));
-
- InMemoryOutputStream stream;
- std::unique_ptr<Int64Writer> writer = BuildWriter(
- state.range(0), &stream, metadata.get(), schema.get(), properties.get());
- writer->WriteBatch(values.size(), definition_levels.data(), repetition_levels.data(),
- values.data());
- writer->Close();
-
- std::shared_ptr<Buffer> src = stream.GetBuffer();
- std::vector<int64_t> values_out(state.range(1));
- std::vector<int16_t> definition_levels_out(state.range(1));
- std::vector<int16_t> repetition_levels_out(state.range(1));
- while (state.KeepRunning()) {
- std::unique_ptr<Int64Reader> reader = BuildReader(src, state.range(1), schema.get());
- int64_t values_read = 0;
- for (size_t i = 0; i < values.size(); i += values_read) {
- reader->ReadBatch(values_out.size(), definition_levels_out.data(),
- repetition_levels_out.data(), values_out.data(), &values_read);
- }
- }
- SetBytesProcessed(state, repetition);
-}
-
-BENCHMARK_TEMPLATE(BM_ReadInt64Column, Repetition::REQUIRED)
- ->RangePair(1024, 65536, 1, 1024);
-
-BENCHMARK_TEMPLATE(BM_ReadInt64Column, Repetition::OPTIONAL)
- ->RangePair(1024, 65536, 1, 1024);
-
-BENCHMARK_TEMPLATE(BM_ReadInt64Column, Repetition::REPEATED)
- ->RangePair(1024, 65536, 1, 1024);
-
-BENCHMARK_TEMPLATE(BM_ReadInt64Column, Repetition::REQUIRED, Compression::SNAPPY)
- ->RangePair(1024, 65536, 1, 1024);
-BENCHMARK_TEMPLATE(BM_ReadInt64Column, Repetition::OPTIONAL, Compression::SNAPPY)
- ->RangePair(1024, 65536, 1, 1024);
-BENCHMARK_TEMPLATE(BM_ReadInt64Column, Repetition::REPEATED, Compression::SNAPPY)
- ->RangePair(1024, 65536, 1, 1024);
-
-BENCHMARK_TEMPLATE(BM_ReadInt64Column, Repetition::REQUIRED, Compression::LZ4)
- ->RangePair(1024, 65536, 1, 1024);
-BENCHMARK_TEMPLATE(BM_ReadInt64Column, Repetition::OPTIONAL, Compression::LZ4)
- ->RangePair(1024, 65536, 1, 1024);
-BENCHMARK_TEMPLATE(BM_ReadInt64Column, Repetition::REPEATED, Compression::LZ4)
- ->RangePair(1024, 65536, 1, 1024);
-
-BENCHMARK_TEMPLATE(BM_ReadInt64Column, Repetition::REQUIRED, Compression::ZSTD)
- ->RangePair(1024, 65536, 1, 1024);
-BENCHMARK_TEMPLATE(BM_ReadInt64Column, Repetition::OPTIONAL, Compression::ZSTD)
- ->RangePair(1024, 65536, 1, 1024);
-BENCHMARK_TEMPLATE(BM_ReadInt64Column, Repetition::REPEATED, Compression::ZSTD)
- ->RangePair(1024, 65536, 1, 1024);
-
-static void BM_RleEncoding(::benchmark::State& state) {
- std::vector<int16_t> levels(state.range(0), 0);
- int64_t n = 0;
- std::generate(levels.begin(), levels.end(),
- [&state, &n] { return (n++ % state.range(1)) == 0; });
- int16_t max_level = 1;
- int64_t rle_size = LevelEncoder::MaxBufferSize(Encoding::RLE, max_level,
- static_cast<int>(levels.size()));
- auto buffer_rle = AllocateBuffer();
- PARQUET_THROW_NOT_OK(buffer_rle->Resize(rle_size));
-
- while (state.KeepRunning()) {
- LevelEncoder level_encoder;
- level_encoder.Init(Encoding::RLE, max_level, static_cast<int>(levels.size()),
- buffer_rle->mutable_data(), static_cast<int>(buffer_rle->size()));
- level_encoder.Encode(static_cast<int>(levels.size()), levels.data());
- }
- state.SetBytesProcessed(state.iterations() * state.range(0) * sizeof(int16_t));
- state.SetItemsProcessed(state.iterations() * state.range(0));
-}
-
-BENCHMARK(BM_RleEncoding)->RangePair(1024, 65536, 1, 16);
-
-static void BM_RleDecoding(::benchmark::State& state) {
- LevelEncoder level_encoder;
- std::vector<int16_t> levels(state.range(0), 0);
- int64_t n = 0;
- std::generate(levels.begin(), levels.end(),
- [&state, &n] { return (n++ % state.range(1)) == 0; });
- int16_t max_level = 1;
- int rle_size = LevelEncoder::MaxBufferSize(Encoding::RLE, max_level,
- static_cast<int>(levels.size()));
- auto buffer_rle = AllocateBuffer();
- PARQUET_THROW_NOT_OK(buffer_rle->Resize(rle_size + sizeof(int32_t)));
- level_encoder.Init(Encoding::RLE, max_level, static_cast<int>(levels.size()),
- buffer_rle->mutable_data() + sizeof(int32_t), rle_size);
- level_encoder.Encode(static_cast<int>(levels.size()), levels.data());
- reinterpret_cast<int32_t*>(buffer_rle->mutable_data())[0] = level_encoder.len();
-
- while (state.KeepRunning()) {
- LevelDecoder level_decoder;
- level_decoder.SetData(Encoding::RLE, max_level, static_cast<int>(levels.size()),
- buffer_rle->data());
- level_decoder.Decode(state.range(0), levels.data());
- }
-
- state.SetBytesProcessed(state.iterations() * state.range(0) * sizeof(int16_t));
- state.SetItemsProcessed(state.iterations() * state.range(0));
-}
-
-BENCHMARK(BM_RleDecoding)->RangePair(1024, 65536, 1, 16);
-
-} // namespace benchmark
-
-} // namespace parquet
diff --git a/src/parquet/column_page.h b/src/parquet/column_page.h
deleted file mode 100644
index c34eee7..0000000
--- a/src/parquet/column_page.h
+++ /dev/null
@@ -1,173 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-// This module defines an abstract interface for iterating through pages in a
-// Parquet column chunk within a row group. It could be extended in the future
-// to iterate through all data pages in all chunks in a file.
-
-#ifndef PARQUET_COLUMN_PAGE_H
-#define PARQUET_COLUMN_PAGE_H
-
-#include <cstdint>
-#include <memory>
-#include <string>
-
-#include "parquet/statistics.h"
-#include "parquet/types.h"
-#include "parquet/util/memory.h"
-
-namespace parquet {
-
-// TODO: Parallel processing is not yet safe because of memory-ownership
-// semantics (the PageReader may or may not own the memory referenced by a
-// page)
-//
-// TODO(wesm): In the future Parquet implementations may store the crc code
-// in format::PageHeader. parquet-mr currently does not, so we also skip it
-// here, both on the read and write path
-class Page {
- public:
- Page(const std::shared_ptr<Buffer>& buffer, PageType::type type)
- : buffer_(buffer), type_(type) {}
-
- PageType::type type() const { return type_; }
-
- std::shared_ptr<Buffer> buffer() const { return buffer_; }
-
- // @returns: a pointer to the page's data
- const uint8_t* data() const { return buffer_->data(); }
-
- // @returns: the total size in bytes of the page's data buffer
- int32_t size() const { return static_cast<int32_t>(buffer_->size()); }
-
- private:
- std::shared_ptr<Buffer> buffer_;
- PageType::type type_;
-};
-
-class DataPage : public Page {
- public:
- DataPage(const std::shared_ptr<Buffer>& buffer, int32_t num_values,
- Encoding::type encoding, Encoding::type definition_level_encoding,
- Encoding::type repetition_level_encoding,
- const EncodedStatistics& statistics = EncodedStatistics())
- : Page(buffer, PageType::DATA_PAGE),
- num_values_(num_values),
- encoding_(encoding),
- definition_level_encoding_(definition_level_encoding),
- repetition_level_encoding_(repetition_level_encoding),
- statistics_(statistics) {}
-
- int32_t num_values() const { return num_values_; }
-
- Encoding::type encoding() const { return encoding_; }
-
- Encoding::type repetition_level_encoding() const { return repetition_level_encoding_; }
-
- Encoding::type definition_level_encoding() const { return definition_level_encoding_; }
-
- const EncodedStatistics& statistics() const { return statistics_; }
-
- private:
- int32_t num_values_;
- Encoding::type encoding_;
- Encoding::type definition_level_encoding_;
- Encoding::type repetition_level_encoding_;
- EncodedStatistics statistics_;
-};
-
-class CompressedDataPage : public DataPage {
- public:
- CompressedDataPage(const std::shared_ptr<Buffer>& buffer, int32_t num_values,
- Encoding::type encoding, Encoding::type definition_level_encoding,
- Encoding::type repetition_level_encoding, int64_t uncompressed_size,
- const EncodedStatistics& statistics = EncodedStatistics())
- : DataPage(buffer, num_values, encoding, definition_level_encoding,
- repetition_level_encoding, statistics),
- uncompressed_size_(uncompressed_size) {}
-
- int64_t uncompressed_size() const { return uncompressed_size_; }
-
- private:
- int64_t uncompressed_size_;
-};
-
-class DataPageV2 : public Page {
- public:
- DataPageV2(const std::shared_ptr<Buffer>& buffer, int32_t num_values, int32_t num_nulls,
- int32_t num_rows, Encoding::type encoding,
- int32_t definition_levels_byte_length, int32_t repetition_levels_byte_length,
- bool is_compressed = false)
- : Page(buffer, PageType::DATA_PAGE_V2),
- num_values_(num_values),
- num_nulls_(num_nulls),
- num_rows_(num_rows),
- encoding_(encoding),
- definition_levels_byte_length_(definition_levels_byte_length),
- repetition_levels_byte_length_(repetition_levels_byte_length),
- is_compressed_(is_compressed) {}
-
- int32_t num_values() const { return num_values_; }
-
- int32_t num_nulls() const { return num_nulls_; }
-
- int32_t num_rows() const { return num_rows_; }
-
- Encoding::type encoding() const { return encoding_; }
-
- int32_t definition_levels_byte_length() const { return definition_levels_byte_length_; }
-
- int32_t repetition_levels_byte_length() const { return repetition_levels_byte_length_; }
-
- bool is_compressed() const { return is_compressed_; }
-
- private:
- int32_t num_values_;
- int32_t num_nulls_;
- int32_t num_rows_;
- Encoding::type encoding_;
- int32_t definition_levels_byte_length_;
- int32_t repetition_levels_byte_length_;
- bool is_compressed_;
-
- // TODO(wesm): format::DataPageHeaderV2.statistics
-};
-
-class DictionaryPage : public Page {
- public:
- DictionaryPage(const std::shared_ptr<Buffer>& buffer, int32_t num_values,
- Encoding::type encoding, bool is_sorted = false)
- : Page(buffer, PageType::DICTIONARY_PAGE),
- num_values_(num_values),
- encoding_(encoding),
- is_sorted_(is_sorted) {}
-
- int32_t num_values() const { return num_values_; }
-
- Encoding::type encoding() const { return encoding_; }
-
- bool is_sorted() const { return is_sorted_; }
-
- private:
- int32_t num_values_;
- Encoding::type encoding_;
- bool is_sorted_;
-};
-
-} // namespace parquet
-
-#endif // PARQUET_COLUMN_PAGE_H
diff --git a/src/parquet/column_reader-test.cc b/src/parquet/column_reader-test.cc
deleted file mode 100644
index 273b302..0000000
--- a/src/parquet/column_reader-test.cc
+++ /dev/null
@@ -1,390 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#include <gtest/gtest.h>
-
-#include <algorithm>
-#include <cstdint>
-#include <cstdlib>
-#include <limits>
-#include <memory>
-#include <string>
-#include <vector>
-
-#include "parquet/column_page.h"
-#include "parquet/column_reader.h"
-#include "parquet/schema.h"
-#include "parquet/test-util.h"
-#include "parquet/types.h"
-#include "parquet/util/test-common.h"
-
-using std::shared_ptr;
-using std::string;
-using std::vector;
-
-namespace parquet {
-
-using schema::NodePtr;
-
-namespace test {
-
-template <typename T>
-static inline bool vector_equal_with_def_levels(const vector<T>& left,
- const vector<int16_t>& def_levels,
- int16_t max_def_levels,
- int16_t max_rep_levels,
- const vector<T>& right) {
- size_t i_left = 0;
- size_t i_right = 0;
- for (size_t i = 0; i < def_levels.size(); i++) {
- if (def_levels[i] == max_def_levels) {
- // Compare
- if (left[i_left] != right[i_right]) {
- std::cerr << "index " << i << " left was " << left[i_left] << " right was "
- << right[i] << std::endl;
- return false;
- }
- i_left++;
- i_right++;
- } else if (def_levels[i] == (max_def_levels - 1)) {
- // Null entry on the lowest nested level
- i_right++;
- } else if (def_levels[i] < (max_def_levels - 1)) {
- // Null entry on a higher nesting level, only supported for non-repeating data
- if (max_rep_levels == 0) {
- i_right++;
- }
- }
- }
-
- return true;
-}
-
-class TestPrimitiveReader : public ::testing::Test {
- public:
- void InitReader(const ColumnDescriptor* d) {
- std::unique_ptr<PageReader> pager_;
- pager_.reset(new test::MockPageReader(pages_));
- reader_ = ColumnReader::Make(d, std::move(pager_));
- }
-
- void CheckResults() {
- vector<int32_t> vresult(num_values_, -1);
- vector<int16_t> dresult(num_levels_, -1);
- vector<int16_t> rresult(num_levels_, -1);
- int64_t values_read = 0;
- int total_values_read = 0;
- int batch_actual = 0;
-
- Int32Reader* reader = static_cast<Int32Reader*>(reader_.get());
- int32_t batch_size = 8;
- int batch = 0;
- // This will cover both the cases
- // 1) batch_size < page_size (multiple ReadBatch from a single page)
- // 2) batch_size > page_size (BatchRead limits to a single page)
- do {
- batch = static_cast<int>(reader->ReadBatch(
- batch_size, &dresult[0] + batch_actual, &rresult[0] + batch_actual,
- &vresult[0] + total_values_read, &values_read));
- total_values_read += static_cast<int>(values_read);
- batch_actual += batch;
- batch_size = std::max(batch_size * 2, 4096);
- } while (batch > 0);
-
- ASSERT_EQ(num_levels_, batch_actual);
- ASSERT_EQ(num_values_, total_values_read);
- ASSERT_TRUE(vector_equal(values_, vresult));
- if (max_def_level_ > 0) {
- ASSERT_TRUE(vector_equal(def_levels_, dresult));
- }
- if (max_rep_level_ > 0) {
- ASSERT_TRUE(vector_equal(rep_levels_, rresult));
- }
- // catch improper writes at EOS
- batch_actual =
- static_cast<int>(reader->ReadBatch(5, nullptr, nullptr, nullptr, &values_read));
- ASSERT_EQ(0, batch_actual);
- ASSERT_EQ(0, values_read);
- }
-
- void CheckResultsSpaced() {
- vector<int32_t> vresult(num_levels_, -1);
- vector<int16_t> dresult(num_levels_, -1);
- vector<int16_t> rresult(num_levels_, -1);
- vector<uint8_t> valid_bits(num_levels_, 255);
- int total_values_read = 0;
- int batch_actual = 0;
- int levels_actual = 0;
- int64_t null_count = -1;
- int64_t levels_read = 0;
- int64_t values_read;
-
- Int32Reader* reader = static_cast<Int32Reader*>(reader_.get());
- int32_t batch_size = 8;
- int batch = 0;
- // This will cover both the cases
- // 1) batch_size < page_size (multiple ReadBatch from a single page)
- // 2) batch_size > page_size (BatchRead limits to a single page)
- do {
- batch = static_cast<int>(reader->ReadBatchSpaced(
- batch_size, dresult.data() + levels_actual, rresult.data() + levels_actual,
- vresult.data() + batch_actual, valid_bits.data() + batch_actual, 0,
- &levels_read, &values_read, &null_count));
- total_values_read += batch - static_cast<int>(null_count);
- batch_actual += batch;
- levels_actual += static_cast<int>(levels_read);
- batch_size = std::max(batch_size * 2, 4096);
- } while ((batch > 0) || (levels_read > 0));
-
- ASSERT_EQ(num_levels_, levels_actual);
- ASSERT_EQ(num_values_, total_values_read);
- if (max_def_level_ > 0) {
- ASSERT_TRUE(vector_equal(def_levels_, dresult));
- ASSERT_TRUE(vector_equal_with_def_levels(values_, dresult, max_def_level_,
- max_rep_level_, vresult));
- } else {
- ASSERT_TRUE(vector_equal(values_, vresult));
- }
- if (max_rep_level_ > 0) {
- ASSERT_TRUE(vector_equal(rep_levels_, rresult));
- }
- // catch improper writes at EOS
- batch_actual = static_cast<int>(
- reader->ReadBatchSpaced(5, nullptr, nullptr, nullptr, valid_bits.data(), 0,
- &levels_read, &values_read, &null_count));
- ASSERT_EQ(0, batch_actual);
- ASSERT_EQ(0, null_count);
- }
-
- void Clear() {
- values_.clear();
- def_levels_.clear();
- rep_levels_.clear();
- pages_.clear();
- reader_.reset();
- }
-
- void ExecutePlain(int num_pages, int levels_per_page, const ColumnDescriptor* d) {
- num_values_ =
- MakePages<Int32Type>(d, num_pages, levels_per_page, def_levels_, rep_levels_,
- values_, data_buffer_, pages_, Encoding::PLAIN);
- num_levels_ = num_pages * levels_per_page;
- InitReader(d);
- CheckResults();
- Clear();
-
- num_values_ =
- MakePages<Int32Type>(d, num_pages, levels_per_page, def_levels_, rep_levels_,
- values_, data_buffer_, pages_, Encoding::PLAIN);
- num_levels_ = num_pages * levels_per_page;
- InitReader(d);
- CheckResultsSpaced();
- Clear();
- }
-
- void ExecuteDict(int num_pages, int levels_per_page, const ColumnDescriptor* d) {
- num_values_ =
- MakePages<Int32Type>(d, num_pages, levels_per_page, def_levels_, rep_levels_,
- values_, data_buffer_, pages_, Encoding::RLE_DICTIONARY);
- num_levels_ = num_pages * levels_per_page;
- InitReader(d);
- CheckResults();
- Clear();
-
- num_values_ =
- MakePages<Int32Type>(d, num_pages, levels_per_page, def_levels_, rep_levels_,
- values_, data_buffer_, pages_, Encoding::RLE_DICTIONARY);
- num_levels_ = num_pages * levels_per_page;
- InitReader(d);
- CheckResultsSpaced();
- Clear();
- }
-
- protected:
- int num_levels_;
- int num_values_;
- int16_t max_def_level_;
- int16_t max_rep_level_;
- vector<shared_ptr<Page>> pages_;
- std::shared_ptr<ColumnReader> reader_;
- vector<int32_t> values_;
- vector<int16_t> def_levels_;
- vector<int16_t> rep_levels_;
- vector<uint8_t> data_buffer_; // For BA and FLBA
-};
-
-TEST_F(TestPrimitiveReader, TestInt32FlatRequired) {
- int levels_per_page = 100;
- int num_pages = 50;
- max_def_level_ = 0;
- max_rep_level_ = 0;
- NodePtr type = schema::Int32("a", Repetition::REQUIRED);
- const ColumnDescriptor descr(type, max_def_level_, max_rep_level_);
- ASSERT_NO_FATAL_FAILURE(ExecutePlain(num_pages, levels_per_page, &descr));
- ASSERT_NO_FATAL_FAILURE(ExecuteDict(num_pages, levels_per_page, &descr));
-}
-
-TEST_F(TestPrimitiveReader, TestInt32FlatOptional) {
- int levels_per_page = 100;
- int num_pages = 50;
- max_def_level_ = 4;
- max_rep_level_ = 0;
- NodePtr type = schema::Int32("b", Repetition::OPTIONAL);
- const ColumnDescriptor descr(type, max_def_level_, max_rep_level_);
- ASSERT_NO_FATAL_FAILURE(ExecutePlain(num_pages, levels_per_page, &descr));
- ASSERT_NO_FATAL_FAILURE(ExecuteDict(num_pages, levels_per_page, &descr));
-}
-
-TEST_F(TestPrimitiveReader, TestInt32FlatRepeated) {
- int levels_per_page = 100;
- int num_pages = 50;
- max_def_level_ = 4;
- max_rep_level_ = 2;
- NodePtr type = schema::Int32("c", Repetition::REPEATED);
- const ColumnDescriptor descr(type, max_def_level_, max_rep_level_);
- ASSERT_NO_FATAL_FAILURE(ExecutePlain(num_pages, levels_per_page, &descr));
- ASSERT_NO_FATAL_FAILURE(ExecuteDict(num_pages, levels_per_page, &descr));
-}
-
-TEST_F(TestPrimitiveReader, TestInt32FlatRequiredSkip) {
- int levels_per_page = 100;
- int num_pages = 5;
- max_def_level_ = 0;
- max_rep_level_ = 0;
- NodePtr type = schema::Int32("b", Repetition::REQUIRED);
- const ColumnDescriptor descr(type, max_def_level_, max_rep_level_);
- MakePages<Int32Type>(&descr, num_pages, levels_per_page, def_levels_, rep_levels_,
- values_, data_buffer_, pages_, Encoding::PLAIN);
- InitReader(&descr);
- vector<int32_t> vresult(levels_per_page / 2, -1);
- vector<int16_t> dresult(levels_per_page / 2, -1);
- vector<int16_t> rresult(levels_per_page / 2, -1);
-
- Int32Reader* reader = static_cast<Int32Reader*>(reader_.get());
- int64_t values_read = 0;
-
- // 1) skip_size > page_size (multiple pages skipped)
- // Skip first 2 pages
- int64_t levels_skipped = reader->Skip(2 * levels_per_page);
- ASSERT_EQ(2 * levels_per_page, levels_skipped);
- // Read half a page
- reader->ReadBatch(levels_per_page / 2, dresult.data(), rresult.data(), vresult.data(),
- &values_read);
- vector<int32_t> sub_values(
- values_.begin() + 2 * levels_per_page,
- values_.begin() + static_cast<int>(2.5 * static_cast<double>(levels_per_page)));
- ASSERT_TRUE(vector_equal(sub_values, vresult));
-
- // 2) skip_size == page_size (skip across two pages)
- levels_skipped = reader->Skip(levels_per_page);
- ASSERT_EQ(levels_per_page, levels_skipped);
- // Read half a page
- reader->ReadBatch(levels_per_page / 2, dresult.data(), rresult.data(), vresult.data(),
- &values_read);
- sub_values.clear();
- sub_values.insert(
- sub_values.end(),
- values_.begin() + static_cast<int>(3.5 * static_cast<double>(levels_per_page)),
- values_.begin() + 4 * levels_per_page);
- ASSERT_TRUE(vector_equal(sub_values, vresult));
-
- // 3) skip_size < page_size (skip limited to a single page)
- // Skip half a page
- levels_skipped = reader->Skip(levels_per_page / 2);
- ASSERT_EQ(0.5 * levels_per_page, levels_skipped);
- // Read half a page
- reader->ReadBatch(levels_per_page / 2, dresult.data(), rresult.data(), vresult.data(),
- &values_read);
- sub_values.clear();
- sub_values.insert(
- sub_values.end(),
- values_.begin() + static_cast<int>(4.5 * static_cast<double>(levels_per_page)),
- values_.end());
- ASSERT_TRUE(vector_equal(sub_values, vresult));
-
- values_.clear();
- def_levels_.clear();
- rep_levels_.clear();
- pages_.clear();
- reader_.reset();
-}
-
-TEST_F(TestPrimitiveReader, TestDictionaryEncodedPages) {
- max_def_level_ = 0;
- max_rep_level_ = 0;
- NodePtr type = schema::Int32("a", Repetition::REQUIRED);
- const ColumnDescriptor descr(type, max_def_level_, max_rep_level_);
- shared_ptr<ResizableBuffer> dummy = AllocateBuffer();
-
- shared_ptr<DictionaryPage> dict_page =
- std::make_shared<DictionaryPage>(dummy, 0, Encoding::PLAIN);
- shared_ptr<DataPage> data_page = MakeDataPage<Int32Type>(
- &descr, {}, 0, Encoding::RLE_DICTIONARY, {}, 0, {}, 0, {}, 0);
- pages_.push_back(dict_page);
- pages_.push_back(data_page);
- InitReader(&descr);
- // Tests Dict : PLAIN, Data : RLE_DICTIONARY
- ASSERT_NO_THROW(reader_->HasNext());
- pages_.clear();
-
- dict_page = std::make_shared<DictionaryPage>(dummy, 0, Encoding::PLAIN_DICTIONARY);
- data_page = MakeDataPage<Int32Type>(&descr, {}, 0, Encoding::PLAIN_DICTIONARY, {}, 0,
- {}, 0, {}, 0);
- pages_.push_back(dict_page);
- pages_.push_back(data_page);
- InitReader(&descr);
- // Tests Dict : PLAIN_DICTIONARY, Data : PLAIN_DICTIONARY
- ASSERT_NO_THROW(reader_->HasNext());
- pages_.clear();
-
- data_page = MakeDataPage<Int32Type>(&descr, {}, 0, Encoding::RLE_DICTIONARY, {}, 0, {},
- 0, {}, 0);
- pages_.push_back(data_page);
- InitReader(&descr);
- // Tests dictionary page must occur before data page
- ASSERT_THROW(reader_->HasNext(), ParquetException);
- pages_.clear();
-
- dict_page = std::make_shared<DictionaryPage>(dummy, 0, Encoding::DELTA_BYTE_ARRAY);
- pages_.push_back(dict_page);
- InitReader(&descr);
- // Tests only RLE_DICTIONARY is supported
- ASSERT_THROW(reader_->HasNext(), ParquetException);
- pages_.clear();
-
- shared_ptr<DictionaryPage> dict_page1 =
- std::make_shared<DictionaryPage>(dummy, 0, Encoding::PLAIN_DICTIONARY);
- shared_ptr<DictionaryPage> dict_page2 =
- std::make_shared<DictionaryPage>(dummy, 0, Encoding::PLAIN);
- pages_.push_back(dict_page1);
- pages_.push_back(dict_page2);
- InitReader(&descr);
- // Column cannot have more than one dictionary
- ASSERT_THROW(reader_->HasNext(), ParquetException);
- pages_.clear();
-
- data_page = MakeDataPage<Int32Type>(&descr, {}, 0, Encoding::DELTA_BYTE_ARRAY, {}, 0,
- {}, 0, {}, 0);
- pages_.push_back(data_page);
- InitReader(&descr);
- // unsupported encoding
- ASSERT_THROW(reader_->HasNext(), ParquetException);
- pages_.clear();
-}
-
-} // namespace test
-} // namespace parquet
diff --git a/src/parquet/column_reader.cc b/src/parquet/column_reader.cc
deleted file mode 100644
index 28d0dcb..0000000
--- a/src/parquet/column_reader.cc
+++ /dev/null
@@ -1,475 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#include "parquet/column_reader.h"
-
-#include <algorithm>
-#include <cstdint>
-#include <memory>
-
-#include <arrow/buffer.h>
-#include <arrow/memory_pool.h>
-#include <arrow/util/bit-util.h>
-#include <arrow/util/compression.h>
-#include <arrow/util/rle-encoding.h>
-
-#include "parquet/column_page.h"
-#include "parquet/encoding-internal.h"
-#include "parquet/parquet_types.h"
-#include "parquet/properties.h"
-#include "parquet/thrift.h"
-
-using arrow::MemoryPool;
-
-namespace parquet {
-
-LevelDecoder::LevelDecoder() : num_values_remaining_(0) {}
-
-LevelDecoder::~LevelDecoder() {}
-
-int LevelDecoder::SetData(Encoding::type encoding, int16_t max_level,
- int num_buffered_values, const uint8_t* data) {
- int32_t num_bytes = 0;
- encoding_ = encoding;
- num_values_remaining_ = num_buffered_values;
- bit_width_ = BitUtil::Log2(max_level + 1);
- switch (encoding) {
- case Encoding::RLE: {
- num_bytes = *reinterpret_cast<const int32_t*>(data);
- const uint8_t* decoder_data = data + sizeof(int32_t);
- if (!rle_decoder_) {
- rle_decoder_.reset(new ::arrow::RleDecoder(decoder_data, num_bytes, bit_width_));
- } else {
- rle_decoder_->Reset(decoder_data, num_bytes, bit_width_);
- }
- return static_cast<int>(sizeof(int32_t)) + num_bytes;
- }
- case Encoding::BIT_PACKED: {
- num_bytes =
- static_cast<int32_t>(BitUtil::BytesForBits(num_buffered_values * bit_width_));
- if (!bit_packed_decoder_) {
- bit_packed_decoder_.reset(new ::arrow::BitReader(data, num_bytes));
- } else {
- bit_packed_decoder_->Reset(data, num_bytes);
- }
- return num_bytes;
- }
- default:
- throw ParquetException("Unknown encoding type for levels.");
- }
- return -1;
-}
-
-int LevelDecoder::Decode(int batch_size, int16_t* levels) {
- int num_decoded = 0;
-
- int num_values = std::min(num_values_remaining_, batch_size);
- if (encoding_ == Encoding::RLE) {
- num_decoded = rle_decoder_->GetBatch(levels, num_values);
- } else {
- num_decoded = bit_packed_decoder_->GetBatch(bit_width_, levels, num_values);
- }
- num_values_remaining_ -= num_decoded;
- return num_decoded;
-}
-
-ReaderProperties default_reader_properties() {
- static ReaderProperties default_reader_properties;
- return default_reader_properties;
-}
-
-// ----------------------------------------------------------------------
-// SerializedPageReader deserializes Thrift metadata and pages that have been
-// assembled in a serialized stream for storing in a Parquet files
-
-// This subclass delimits pages appearing in a serialized stream, each preceded
-// by a serialized Thrift format::PageHeader indicating the type of each page
-// and the page metadata.
-class SerializedPageReader : public PageReader {
- public:
- SerializedPageReader(std::unique_ptr<InputStream> stream, int64_t total_num_rows,
- Compression::type codec, ::arrow::MemoryPool* pool)
- : stream_(std::move(stream)),
- decompression_buffer_(AllocateBuffer(pool, 0)),
- seen_num_rows_(0),
- total_num_rows_(total_num_rows) {
- max_page_header_size_ = kDefaultMaxPageHeaderSize;
- decompressor_ = GetCodecFromArrow(codec);
- }
-
- // Implement the PageReader interface
- std::shared_ptr<Page> NextPage() override;
-
- void set_max_page_header_size(uint32_t size) override { max_page_header_size_ = size; }
-
- private:
- std::unique_ptr<InputStream> stream_;
-
- format::PageHeader current_page_header_;
- std::shared_ptr<Page> current_page_;
-
- // Compression codec to use.
- std::unique_ptr<::arrow::Codec> decompressor_;
- std::shared_ptr<ResizableBuffer> decompression_buffer_;
-
- // Maximum allowed page size
- uint32_t max_page_header_size_;
-
- // Number of rows read in data pages so far
- int64_t seen_num_rows_;
-
- // Number of rows in all the data pages
- int64_t total_num_rows_;
-};
-
-std::shared_ptr<Page> SerializedPageReader::NextPage() {
- // Loop here because there may be unhandled page types that we skip until
- // finding a page that we do know what to do with
- while (seen_num_rows_ < total_num_rows_) {
- int64_t bytes_read = 0;
- int64_t bytes_available = 0;
- uint32_t header_size = 0;
- const uint8_t* buffer;
- uint32_t allowed_page_size = kDefaultPageHeaderSize;
-
- // Page headers can be very large because of page statistics
- // We try to deserialize a larger buffer progressively
- // until a maximum allowed header limit
- while (true) {
- buffer = stream_->Peek(allowed_page_size, &bytes_available);
- if (bytes_available == 0) {
- return std::shared_ptr<Page>(nullptr);
- }
-
- // This gets used, then set by DeserializeThriftMsg
- header_size = static_cast<uint32_t>(bytes_available);
- try {
- DeserializeThriftMsg(buffer, &header_size, ¤t_page_header_);
- break;
- } catch (std::exception& e) {
- // Failed to deserialize. Double the allowed page header size and try again
- std::stringstream ss;
- ss << e.what();
- allowed_page_size *= 2;
- if (allowed_page_size > max_page_header_size_) {
- ss << "Deserializing page header failed.\n";
- throw ParquetException(ss.str());
- }
- }
- }
- // Advance the stream offset
- stream_->Advance(header_size);
-
- int compressed_len = current_page_header_.compressed_page_size;
- int uncompressed_len = current_page_header_.uncompressed_page_size;
-
- // Read the compressed data page.
- buffer = stream_->Read(compressed_len, &bytes_read);
- if (bytes_read != compressed_len) {
- std::stringstream ss;
- ss << "Page was smaller (" << bytes_read << ") than expected (" << compressed_len
- << ")";
- ParquetException::EofException(ss.str());
- }
-
- // Uncompress it if we need to
- if (decompressor_ != nullptr) {
- // Grow the uncompressed buffer if we need to.
- if (uncompressed_len > static_cast<int>(decompression_buffer_->size())) {
- PARQUET_THROW_NOT_OK(decompression_buffer_->Resize(uncompressed_len, false));
- }
- PARQUET_THROW_NOT_OK(
- decompressor_->Decompress(compressed_len, buffer, uncompressed_len,
- decompression_buffer_->mutable_data()));
- buffer = decompression_buffer_->data();
- }
-
- auto page_buffer = std::make_shared<Buffer>(buffer, uncompressed_len);
-
- if (current_page_header_.type == format::PageType::DICTIONARY_PAGE) {
- const format::DictionaryPageHeader& dict_header =
- current_page_header_.dictionary_page_header;
-
- bool is_sorted = dict_header.__isset.is_sorted ? dict_header.is_sorted : false;
-
- return std::make_shared<DictionaryPage>(page_buffer, dict_header.num_values,
- FromThrift(dict_header.encoding),
- is_sorted);
- } else if (current_page_header_.type == format::PageType::DATA_PAGE) {
- const format::DataPageHeader& header = current_page_header_.data_page_header;
-
- EncodedStatistics page_statistics;
- if (header.__isset.statistics) {
- const format::Statistics& stats = header.statistics;
- if (stats.__isset.max) {
- page_statistics.set_max(stats.max);
- }
- if (stats.__isset.min) {
- page_statistics.set_min(stats.min);
- }
- if (stats.__isset.null_count) {
- page_statistics.set_null_count(stats.null_count);
- }
- if (stats.__isset.distinct_count) {
- page_statistics.set_distinct_count(stats.distinct_count);
- }
- }
-
- seen_num_rows_ += header.num_values;
-
- return std::make_shared<DataPage>(
- page_buffer, header.num_values, FromThrift(header.encoding),
- FromThrift(header.definition_level_encoding),
- FromThrift(header.repetition_level_encoding), page_statistics);
- } else if (current_page_header_.type == format::PageType::DATA_PAGE_V2) {
- const format::DataPageHeaderV2& header = current_page_header_.data_page_header_v2;
- bool is_compressed = header.__isset.is_compressed ? header.is_compressed : false;
-
- seen_num_rows_ += header.num_values;
-
- return std::make_shared<DataPageV2>(
- page_buffer, header.num_values, header.num_nulls, header.num_rows,
- FromThrift(header.encoding), header.definition_levels_byte_length,
- header.repetition_levels_byte_length, is_compressed);
- } else {
- // We don't know what this page type is. We're allowed to skip non-data
- // pages.
- continue;
- }
- }
- return std::shared_ptr<Page>(nullptr);
-}
-
-std::unique_ptr<PageReader> PageReader::Open(std::unique_ptr<InputStream> stream,
- int64_t total_num_rows,
- Compression::type codec,
- ::arrow::MemoryPool* pool) {
- return std::unique_ptr<PageReader>(
- new SerializedPageReader(std::move(stream), total_num_rows, codec, pool));
-}
-
-// ----------------------------------------------------------------------
-
-ColumnReader::ColumnReader(const ColumnDescriptor* descr,
- std::unique_ptr<PageReader> pager, MemoryPool* pool)
- : descr_(descr),
- pager_(std::move(pager)),
- num_buffered_values_(0),
- num_decoded_values_(0),
- pool_(pool) {}
-
-ColumnReader::~ColumnReader() {}
-
-template <typename DType>
-void TypedColumnReader<DType>::ConfigureDictionary(const DictionaryPage* page) {
- int encoding = static_cast<int>(page->encoding());
- if (page->encoding() == Encoding::PLAIN_DICTIONARY ||
- page->encoding() == Encoding::PLAIN) {
- encoding = static_cast<int>(Encoding::RLE_DICTIONARY);
- }
-
- auto it = decoders_.find(encoding);
- if (it != decoders_.end()) {
- throw ParquetException("Column cannot have more than one dictionary.");
- }
-
- if (page->encoding() == Encoding::PLAIN_DICTIONARY ||
- page->encoding() == Encoding::PLAIN) {
- PlainDecoder<DType> dictionary(descr_);
- dictionary.SetData(page->num_values(), page->data(), page->size());
-
- // The dictionary is fully decoded during DictionaryDecoder::Init, so the
- // DictionaryPage buffer is no longer required after this step
- //
- // TODO(wesm): investigate whether this all-or-nothing decoding of the
- // dictionary makes sense and whether performance can be improved
-
- auto decoder = std::make_shared<DictionaryDecoder<DType>>(descr_, pool_);
- decoder->SetDict(&dictionary);
- decoders_[encoding] = decoder;
- } else {
- ParquetException::NYI("only plain dictionary encoding has been implemented");
- }
-
- current_decoder_ = decoders_[encoding].get();
-}
-
-// PLAIN_DICTIONARY is deprecated but used to be used as a dictionary index
-// encoding.
-static bool IsDictionaryIndexEncoding(const Encoding::type& e) {
- return e == Encoding::RLE_DICTIONARY || e == Encoding::PLAIN_DICTIONARY;
-}
-
-template <typename DType>
-bool TypedColumnReader<DType>::ReadNewPage() {
- // Loop until we find the next data page.
- const uint8_t* buffer;
-
- while (true) {
- current_page_ = pager_->NextPage();
- if (!current_page_) {
- // EOS
- return false;
- }
-
- if (current_page_->type() == PageType::DICTIONARY_PAGE) {
- ConfigureDictionary(static_cast<const DictionaryPage*>(current_page_.get()));
- continue;
- } else if (current_page_->type() == PageType::DATA_PAGE) {
- const DataPage* page = static_cast<const DataPage*>(current_page_.get());
-
- // Read a data page.
- num_buffered_values_ = page->num_values();
-
- // Have not decoded any values from the data page yet
- num_decoded_values_ = 0;
-
- buffer = page->data();
-
- // If the data page includes repetition and definition levels, we
- // initialize the level decoder and subtract the encoded level bytes from
- // the page size to determine the number of bytes in the encoded data.
- int64_t data_size = page->size();
-
- // Data page Layout: Repetition Levels - Definition Levels - encoded values.
- // Levels are encoded as rle or bit-packed.
- // Init repetition levels
- if (descr_->max_repetition_level() > 0) {
- int64_t rep_levels_bytes = repetition_level_decoder_.SetData(
- page->repetition_level_encoding(), descr_->max_repetition_level(),
- static_cast<int>(num_buffered_values_), buffer);
- buffer += rep_levels_bytes;
- data_size -= rep_levels_bytes;
- }
- // TODO figure a way to set max_definition_level_ to 0
- // if the initial value is invalid
-
- // Init definition levels
- if (descr_->max_definition_level() > 0) {
- int64_t def_levels_bytes = definition_level_decoder_.SetData(
- page->definition_level_encoding(), descr_->max_definition_level(),
- static_cast<int>(num_buffered_values_), buffer);
- buffer += def_levels_bytes;
- data_size -= def_levels_bytes;
- }
-
- // Get a decoder object for this page or create a new decoder if this is the
- // first page with this encoding.
- Encoding::type encoding = page->encoding();
-
- if (IsDictionaryIndexEncoding(encoding)) {
- encoding = Encoding::RLE_DICTIONARY;
- }
-
- auto it = decoders_.find(static_cast<int>(encoding));
- if (it != decoders_.end()) {
- if (encoding == Encoding::RLE_DICTIONARY) {
- DCHECK(current_decoder_->encoding() == Encoding::RLE_DICTIONARY);
- }
- current_decoder_ = it->second.get();
- } else {
- switch (encoding) {
- case Encoding::PLAIN: {
- std::shared_ptr<DecoderType> decoder(new PlainDecoder<DType>(descr_));
- decoders_[static_cast<int>(encoding)] = decoder;
- current_decoder_ = decoder.get();
- break;
- }
- case Encoding::RLE_DICTIONARY:
- throw ParquetException("Dictionary page must be before data page.");
-
- case Encoding::DELTA_BINARY_PACKED:
- case Encoding::DELTA_LENGTH_BYTE_ARRAY:
- case Encoding::DELTA_BYTE_ARRAY:
- ParquetException::NYI("Unsupported encoding");
-
- default:
- throw ParquetException("Unknown encoding type.");
- }
- }
- current_decoder_->SetData(static_cast<int>(num_buffered_values_), buffer,
- static_cast<int>(data_size));
- return true;
- } else {
- // We don't know what this page type is. We're allowed to skip non-data
- // pages.
- continue;
- }
- }
- return true;
-}
-
-// ----------------------------------------------------------------------
-// Batch read APIs
-
-int64_t ColumnReader::ReadDefinitionLevels(int64_t batch_size, int16_t* levels) {
- if (descr_->max_definition_level() == 0) {
- return 0;
- }
- return definition_level_decoder_.Decode(static_cast<int>(batch_size), levels);
-}
-
-int64_t ColumnReader::ReadRepetitionLevels(int64_t batch_size, int16_t* levels) {
- if (descr_->max_repetition_level() == 0) {
- return 0;
- }
- return repetition_level_decoder_.Decode(static_cast<int>(batch_size), levels);
-}
-
-// ----------------------------------------------------------------------
-// Dynamic column reader constructor
-
-std::shared_ptr<ColumnReader> ColumnReader::Make(const ColumnDescriptor* descr,
- std::unique_ptr<PageReader> pager,
- MemoryPool* pool) {
- switch (descr->physical_type()) {
- case Type::BOOLEAN:
- return std::make_shared<BoolReader>(descr, std::move(pager), pool);
- case Type::INT32:
- return std::make_shared<Int32Reader>(descr, std::move(pager), pool);
- case Type::INT64:
- return std::make_shared<Int64Reader>(descr, std::move(pager), pool);
- case Type::INT96:
- return std::make_shared<Int96Reader>(descr, std::move(pager), pool);
- case Type::FLOAT:
- return std::make_shared<FloatReader>(descr, std::move(pager), pool);
- case Type::DOUBLE:
- return std::make_shared<DoubleReader>(descr, std::move(pager), pool);
- case Type::BYTE_ARRAY:
- return std::make_shared<ByteArrayReader>(descr, std::move(pager), pool);
- case Type::FIXED_LEN_BYTE_ARRAY:
- return std::make_shared<FixedLenByteArrayReader>(descr, std::move(pager), pool);
- default:
- ParquetException::NYI("type reader not implemented");
- }
- // Unreachable code, but supress compiler warning
- return std::shared_ptr<ColumnReader>(nullptr);
-}
-
-// ----------------------------------------------------------------------
-// Instantiate templated classes
-
-template class PARQUET_TEMPLATE_EXPORT TypedColumnReader<BooleanType>;
-template class PARQUET_TEMPLATE_EXPORT TypedColumnReader<Int32Type>;
-template class PARQUET_TEMPLATE_EXPORT TypedColumnReader<Int64Type>;
-template class PARQUET_TEMPLATE_EXPORT TypedColumnReader<Int96Type>;
-template class PARQUET_TEMPLATE_EXPORT TypedColumnReader<FloatType>;
-template class PARQUET_TEMPLATE_EXPORT TypedColumnReader<DoubleType>;
-template class PARQUET_TEMPLATE_EXPORT TypedColumnReader<ByteArrayType>;
-template class PARQUET_TEMPLATE_EXPORT TypedColumnReader<FLBAType>;
-
-} // namespace parquet
diff --git a/src/parquet/column_reader.h b/src/parquet/column_reader.h
deleted file mode 100644
index 7134632..0000000
--- a/src/parquet/column_reader.h
+++ /dev/null
@@ -1,535 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#ifndef PARQUET_COLUMN_READER_H
-#define PARQUET_COLUMN_READER_H
-
-#include <algorithm>
-#include <climits>
-#include <cstdint>
-#include <cstring>
-#include <iostream>
-#include <memory>
-#include <unordered_map>
-#include <vector>
-
-#include <arrow/buffer.h>
-#include <arrow/builder.h>
-#include <arrow/memory_pool.h>
-#include <arrow/util/bit-util.h>
-
-#include "parquet/column_page.h"
-#include "parquet/encoding.h"
-#include "parquet/exception.h"
-#include "parquet/schema.h"
-#include "parquet/types.h"
-#include "parquet/util/memory.h"
-#include "parquet/util/visibility.h"
-
-namespace arrow {
-
-class BitReader;
-class RleDecoder;
-
-} // namespace arrow
-
-namespace parquet {
-
-// 16 MB is the default maximum page header size
-static constexpr uint32_t kDefaultMaxPageHeaderSize = 16 * 1024 * 1024;
-
-// 16 KB is the default expected page header size
-static constexpr uint32_t kDefaultPageHeaderSize = 16 * 1024;
-
-namespace BitUtil = ::arrow::BitUtil;
-
-class PARQUET_EXPORT LevelDecoder {
- public:
- LevelDecoder();
- ~LevelDecoder();
-
- // Initialize the LevelDecoder state with new data
- // and return the number of bytes consumed
- int SetData(Encoding::type encoding, int16_t max_level, int num_buffered_values,
- const uint8_t* data);
-
- // Decodes a batch of levels into an array and returns the number of levels decoded
- int Decode(int batch_size, int16_t* levels);
-
- private:
- int bit_width_;
- int num_values_remaining_;
- Encoding::type encoding_;
- std::unique_ptr<::arrow::RleDecoder> rle_decoder_;
- std::unique_ptr<::arrow::BitReader> bit_packed_decoder_;
-};
-
-// Abstract page iterator interface. This way, we can feed column pages to the
-// ColumnReader through whatever mechanism we choose
-class PARQUET_EXPORT PageReader {
- public:
- virtual ~PageReader() = default;
-
- static std::unique_ptr<PageReader> Open(
- std::unique_ptr<InputStream> stream, int64_t total_num_rows,
- Compression::type codec,
- ::arrow::MemoryPool* pool = ::arrow::default_memory_pool());
-
- // @returns: shared_ptr<Page>(nullptr) on EOS, std::shared_ptr<Page>
- // containing new Page otherwise
- virtual std::shared_ptr<Page> NextPage() = 0;
-
- virtual void set_max_page_header_size(uint32_t size) = 0;
-};
-
-class PARQUET_EXPORT ColumnReader {
- public:
- ColumnReader(const ColumnDescriptor*, std::unique_ptr<PageReader>,
- ::arrow::MemoryPool* pool = ::arrow::default_memory_pool());
- virtual ~ColumnReader();
-
- static std::shared_ptr<ColumnReader> Make(
- const ColumnDescriptor* descr, std::unique_ptr<PageReader> pager,
- ::arrow::MemoryPool* pool = ::arrow::default_memory_pool());
-
- // Returns true if there are still values in this column.
- bool HasNext() {
- // Either there is no data page available yet, or the data page has been
- // exhausted
- if (num_buffered_values_ == 0 || num_decoded_values_ == num_buffered_values_) {
- if (!ReadNewPage() || num_buffered_values_ == 0) {
- return false;
- }
- }
- return true;
- }
-
- Type::type type() const { return descr_->physical_type(); }
-
- const ColumnDescriptor* descr() const { return descr_; }
-
- protected:
- virtual bool ReadNewPage() = 0;
-
- // Read multiple definition levels into preallocated memory
- //
- // Returns the number of decoded definition levels
- int64_t ReadDefinitionLevels(int64_t batch_size, int16_t* levels);
-
- // Read multiple repetition levels into preallocated memory
- // Returns the number of decoded repetition levels
- int64_t ReadRepetitionLevels(int64_t batch_size, int16_t* levels);
-
- int64_t available_values_current_page() const {
- return num_buffered_values_ - num_decoded_values_;
- }
-
- void ConsumeBufferedValues(int64_t num_values) { num_decoded_values_ += num_values; }
-
- const ColumnDescriptor* descr_;
-
- std::unique_ptr<PageReader> pager_;
- std::shared_ptr<Page> current_page_;
-
- // Not set if full schema for this field has no optional or repeated elements
- LevelDecoder definition_level_decoder_;
-
- // Not set for flat schemas.
- LevelDecoder repetition_level_decoder_;
-
- // The total number of values stored in the data page. This is the maximum of
- // the number of encoded definition levels or encoded values. For
- // non-repeated, required columns, this is equal to the number of encoded
- // values. For repeated or optional values, there may be fewer data values
- // than levels, and this tells you how many encoded levels there are in that
- // case.
- int64_t num_buffered_values_;
-
- // The number of values from the current data page that have been decoded
- // into memory
- int64_t num_decoded_values_;
-
- ::arrow::MemoryPool* pool_;
-};
-
-namespace internal {
-
-static inline void DefinitionLevelsToBitmap(
- const int16_t* def_levels, int64_t num_def_levels, const int16_t max_definition_level,
- const int16_t max_repetition_level, int64_t* values_read, int64_t* null_count,
- uint8_t* valid_bits, const int64_t valid_bits_offset) {
- ::arrow::internal::BitmapWriter valid_bits_writer(valid_bits, valid_bits_offset,
- num_def_levels);
-
- // TODO(itaiin): As an interim solution we are splitting the code path here
- // between repeated+flat column reads, and non-repeated+nested reads.
- // Those paths need to be merged in the future
- for (int i = 0; i < num_def_levels; ++i) {
- if (def_levels[i] == max_definition_level) {
- valid_bits_writer.Set();
- } else if (max_repetition_level > 0) {
- // repetition+flat case
- if (def_levels[i] == (max_definition_level - 1)) {
- valid_bits_writer.Clear();
- *null_count += 1;
- } else {
- continue;
- }
- } else {
- // non-repeated+nested case
- if (def_levels[i] < max_definition_level) {
- valid_bits_writer.Clear();
- *null_count += 1;
- } else {
- throw ParquetException("definition level exceeds maximum");
- }
- }
-
- valid_bits_writer.Next();
- }
- valid_bits_writer.Finish();
- *values_read = valid_bits_writer.position();
-}
-
-} // namespace internal
-
-// API to read values from a single column. This is a main client facing API.
-template <typename DType>
-class PARQUET_EXPORT TypedColumnReader : public ColumnReader {
- public:
- typedef typename DType::c_type T;
-
- TypedColumnReader(const ColumnDescriptor* schema, std::unique_ptr<PageReader> pager,
- ::arrow::MemoryPool* pool = ::arrow::default_memory_pool())
- : ColumnReader(schema, std::move(pager), pool), current_decoder_(nullptr) {}
-
- // Read a batch of repetition levels, definition levels, and values from the
- // column.
- //
- // Since null values are not stored in the values, the number of values read
- // may be less than the number of repetition and definition levels. With
- // nested data this is almost certainly true.
- //
- // Set def_levels or rep_levels to nullptr if you want to skip reading them.
- // This is only safe if you know through some other source that there are no
- // undefined values.
- //
- // To fully exhaust a row group, you must read batches until the number of
- // values read reaches the number of stored values according to the metadata.
- //
- // This API is the same for both V1 and V2 of the DataPage
- //
- // @returns: actual number of levels read (see values_read for number of values read)
- int64_t ReadBatch(int64_t batch_size, int16_t* def_levels, int16_t* rep_levels,
- T* values, int64_t* values_read);
-
- /// Read a batch of repetition levels, definition levels, and values from the
- /// column and leave spaces for null entries on the lowest level in the values
- /// buffer.
- ///
- /// In comparision to ReadBatch the length of repetition and definition levels
- /// is the same as of the number of values read for max_definition_level == 1.
- /// In the case of max_definition_level > 1, the repetition and definition
- /// levels are larger than the values but the values include the null entries
- /// with definition_level == (max_definition_level - 1).
- ///
- /// To fully exhaust a row group, you must read batches until the number of
- /// values read reaches the number of stored values according to the metadata.
- ///
- /// @param batch_size the number of levels to read
- /// @param[out] def_levels The Parquet definition levels, output has
- /// the length levels_read.
- /// @param[out] rep_levels The Parquet repetition levels, output has
- /// the length levels_read.
- /// @param[out] values The values in the lowest nested level including
- /// spacing for nulls on the lowest levels; output has the length
- /// values_read.
- /// @param[out] valid_bits Memory allocated for a bitmap that indicates if
- /// the row is null or on the maximum definition level. For performance
- /// reasons the underlying buffer should be able to store 1 bit more than
- /// required. If this requires an additional byte, this byte is only read
- /// but never written to.
- /// @param valid_bits_offset The offset in bits of the valid_bits where the
- /// first relevant bit resides.
- /// @param[out] levels_read The number of repetition/definition levels that were read.
- /// @param[out] values_read The number of values read, this includes all
- /// non-null entries as well as all null-entries on the lowest level
- /// (i.e. definition_level == max_definition_level - 1)
- /// @param[out] null_count The number of nulls on the lowest levels.
- /// (i.e. (values_read - null_count) is total number of non-null entries)
- int64_t ReadBatchSpaced(int64_t batch_size, int16_t* def_levels, int16_t* rep_levels,
- T* values, uint8_t* valid_bits, int64_t valid_bits_offset,
- int64_t* levels_read, int64_t* values_read,
- int64_t* null_count);
-
- // Skip reading levels
- // Returns the number of levels skipped
- int64_t Skip(int64_t num_rows_to_skip);
-
- private:
- typedef Decoder<DType> DecoderType;
-
- // Advance to the next data page
- bool ReadNewPage() override;
-
- // Read up to batch_size values from the current data page into the
- // pre-allocated memory T*
- //
- // @returns: the number of values read into the out buffer
- int64_t ReadValues(int64_t batch_size, T* out);
-
- // Read up to batch_size values from the current data page into the
- // pre-allocated memory T*, leaving spaces for null entries according
- // to the def_levels.
- //
- // @returns: the number of values read into the out buffer
- int64_t ReadValuesSpaced(int64_t batch_size, T* out, int64_t null_count,
- uint8_t* valid_bits, int64_t valid_bits_offset);
-
- // Map of encoding type to the respective decoder object. For example, a
- // column chunk's data pages may include both dictionary-encoded and
- // plain-encoded data.
- std::unordered_map<int, std::shared_ptr<DecoderType>> decoders_;
-
- void ConfigureDictionary(const DictionaryPage* page);
-
- DecoderType* current_decoder_;
-};
-
-// ----------------------------------------------------------------------
-// Type column reader implementations
-
-template <typename DType>
-inline int64_t TypedColumnReader<DType>::ReadValues(int64_t batch_size, T* out) {
- int64_t num_decoded = current_decoder_->Decode(out, static_cast<int>(batch_size));
- return num_decoded;
-}
-
-template <typename DType>
-inline int64_t TypedColumnReader<DType>::ReadValuesSpaced(int64_t batch_size, T* out,
- int64_t null_count,
- uint8_t* valid_bits,
- int64_t valid_bits_offset) {
- return current_decoder_->DecodeSpaced(out, static_cast<int>(batch_size),
- static_cast<int>(null_count), valid_bits,
- valid_bits_offset);
-}
-
-template <typename DType>
-inline int64_t TypedColumnReader<DType>::ReadBatch(int64_t batch_size,
- int16_t* def_levels,
- int16_t* rep_levels, T* values,
- int64_t* values_read) {
- // HasNext invokes ReadNewPage
- if (!HasNext()) {
- *values_read = 0;
- return 0;
- }
-
- // TODO(wesm): keep reading data pages until batch_size is reached, or the
- // row group is finished
- batch_size = std::min(batch_size, num_buffered_values_ - num_decoded_values_);
-
- int64_t num_def_levels = 0;
- int64_t num_rep_levels = 0;
-
- int64_t values_to_read = 0;
-
- // If the field is required and non-repeated, there are no definition levels
- if (descr_->max_definition_level() > 0 && def_levels) {
- num_def_levels = ReadDefinitionLevels(batch_size, def_levels);
- // TODO(wesm): this tallying of values-to-decode can be performed with better
- // cache-efficiency if fused with the level decoding.
- for (int64_t i = 0; i < num_def_levels; ++i) {
- if (def_levels[i] == descr_->max_definition_level()) {
- ++values_to_read;
- }
- }
- } else {
- // Required field, read all values
- values_to_read = batch_size;
- }
-
- // Not present for non-repeated fields
- if (descr_->max_repetition_level() > 0 && rep_levels) {
- num_rep_levels = ReadRepetitionLevels(batch_size, rep_levels);
- if (def_levels && num_def_levels != num_rep_levels) {
- throw ParquetException("Number of decoded rep / def levels did not match");
- }
- }
-
- *values_read = ReadValues(values_to_read, values);
- int64_t total_values = std::max(num_def_levels, *values_read);
- ConsumeBufferedValues(total_values);
-
- return total_values;
-}
-
-namespace internal {
-
-// TODO(itaiin): another code path split to merge when the general case is done
-static inline bool HasSpacedValues(const ColumnDescriptor* descr) {
- if (descr->max_repetition_level() > 0) {
- // repeated+flat case
- return !descr->schema_node()->is_required();
- } else {
- // non-repeated+nested case
- // Find if a node forces nulls in the lowest level along the hierarchy
- const schema::Node* node = descr->schema_node().get();
- while (node) {
- if (node->is_optional()) {
- return true;
- }
- node = node->parent();
- }
- return false;
- }
-}
-
-} // namespace internal
-
-template <typename DType>
-inline int64_t TypedColumnReader<DType>::ReadBatchSpaced(
- int64_t batch_size, int16_t* def_levels, int16_t* rep_levels, T* values,
- uint8_t* valid_bits, int64_t valid_bits_offset, int64_t* levels_read,
- int64_t* values_read, int64_t* null_count_out) {
- // HasNext invokes ReadNewPage
- if (!HasNext()) {
- *levels_read = 0;
- *values_read = 0;
- *null_count_out = 0;
- return 0;
- }
-
- int64_t total_values;
- // TODO(wesm): keep reading data pages until batch_size is reached, or the
- // row group is finished
- batch_size = std::min(batch_size, num_buffered_values_ - num_decoded_values_);
-
- // If the field is required and non-repeated, there are no definition levels
- if (descr_->max_definition_level() > 0) {
- int64_t num_def_levels = ReadDefinitionLevels(batch_size, def_levels);
-
- // Not present for non-repeated fields
- if (descr_->max_repetition_level() > 0) {
- int64_t num_rep_levels = ReadRepetitionLevels(batch_size, rep_levels);
- if (num_def_levels != num_rep_levels) {
- throw ParquetException("Number of decoded rep / def levels did not match");
- }
- }
-
- const bool has_spaced_values = internal::HasSpacedValues(descr_);
-
- int64_t null_count = 0;
- if (!has_spaced_values) {
- int values_to_read = 0;
- for (int64_t i = 0; i < num_def_levels; ++i) {
- if (def_levels[i] == descr_->max_definition_level()) {
- ++values_to_read;
- }
- }
- total_values = ReadValues(values_to_read, values);
- for (int64_t i = 0; i < total_values; i++) {
- ::arrow::BitUtil::SetBit(valid_bits, valid_bits_offset + i);
- }
- *values_read = total_values;
- } else {
- int16_t max_definition_level = descr_->max_definition_level();
- int16_t max_repetition_level = descr_->max_repetition_level();
- internal::DefinitionLevelsToBitmap(def_levels, num_def_levels, max_definition_level,
- max_repetition_level, values_read, &null_count,
- valid_bits, valid_bits_offset);
- total_values = ReadValuesSpaced(*values_read, values, static_cast<int>(null_count),
- valid_bits, valid_bits_offset);
- }
- *levels_read = num_def_levels;
- *null_count_out = null_count;
-
- } else {
- // Required field, read all values
- total_values = ReadValues(batch_size, values);
- for (int64_t i = 0; i < total_values; i++) {
- ::arrow::BitUtil::SetBit(valid_bits, valid_bits_offset + i);
- }
- *null_count_out = 0;
- *levels_read = total_values;
- }
-
- ConsumeBufferedValues(*levels_read);
- return total_values;
-}
-
-template <typename DType>
-int64_t TypedColumnReader<DType>::Skip(int64_t num_rows_to_skip) {
- int64_t rows_to_skip = num_rows_to_skip;
- while (HasNext() && rows_to_skip > 0) {
- // If the number of rows to skip is more than the number of undecoded values, skip the
- // Page.
- if (rows_to_skip > (num_buffered_values_ - num_decoded_values_)) {
- rows_to_skip -= num_buffered_values_ - num_decoded_values_;
- num_decoded_values_ = num_buffered_values_;
- } else {
- // We need to read this Page
- // Jump to the right offset in the Page
- int64_t batch_size = 1024; // ReadBatch with a smaller memory footprint
- int64_t values_read = 0;
-
- std::shared_ptr<ResizableBuffer> vals = AllocateBuffer(
- this->pool_, batch_size * type_traits<DType::type_num>::value_byte_size);
- std::shared_ptr<ResizableBuffer> def_levels =
- AllocateBuffer(this->pool_, batch_size * sizeof(int16_t));
-
- std::shared_ptr<ResizableBuffer> rep_levels =
- AllocateBuffer(this->pool_, batch_size * sizeof(int16_t));
-
- do {
- batch_size = std::min(batch_size, rows_to_skip);
- values_read = ReadBatch(static_cast<int>(batch_size),
- reinterpret_cast<int16_t*>(def_levels->mutable_data()),
- reinterpret_cast<int16_t*>(rep_levels->mutable_data()),
- reinterpret_cast<T*>(vals->mutable_data()), &values_read);
- rows_to_skip -= values_read;
- } while (values_read > 0 && rows_to_skip > 0);
- }
- }
- return num_rows_to_skip - rows_to_skip;
-}
-
-// ----------------------------------------------------------------------
-// Template instantiations
-
-typedef TypedColumnReader<BooleanType> BoolReader;
-typedef TypedColumnReader<Int32Type> Int32Reader;
-typedef TypedColumnReader<Int64Type> Int64Reader;
-typedef TypedColumnReader<Int96Type> Int96Reader;
-typedef TypedColumnReader<FloatType> FloatReader;
-typedef TypedColumnReader<DoubleType> DoubleReader;
-typedef TypedColumnReader<ByteArrayType> ByteArrayReader;
-typedef TypedColumnReader<FLBAType> FixedLenByteArrayReader;
-
-extern template class PARQUET_EXPORT TypedColumnReader<BooleanType>;
-extern template class PARQUET_EXPORT TypedColumnReader<Int32Type>;
-extern template class PARQUET_EXPORT TypedColumnReader<Int64Type>;
-extern template class PARQUET_EXPORT TypedColumnReader<Int96Type>;
-extern template class PARQUET_EXPORT TypedColumnReader<FloatType>;
-extern template class PARQUET_EXPORT TypedColumnReader<DoubleType>;
-extern template class PARQUET_EXPORT TypedColumnReader<ByteArrayType>;
-extern template class PARQUET_EXPORT TypedColumnReader<FLBAType>;
-
-} // namespace parquet
-
-#endif // PARQUET_COLUMN_READER_H
diff --git a/src/parquet/column_scanner-test.cc b/src/parquet/column_scanner-test.cc
deleted file mode 100644
index d39b910..0000000
--- a/src/parquet/column_scanner-test.cc
+++ /dev/null
@@ -1,238 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#include <gtest/gtest.h>
-
-#include <algorithm>
-#include <cstdint>
-#include <cstdlib>
-#include <memory>
-#include <string>
-#include <vector>
-
-#include "parquet/column_page.h"
-#include "parquet/column_scanner.h"
-#include "parquet/schema.h"
-#include "parquet/test-specialization.h"
-#include "parquet/test-util.h"
-#include "parquet/types.h"
-#include "parquet/util/test-common.h"
-
-using std::shared_ptr;
-using std::string;
-using std::vector;
-
-namespace parquet {
-
-using schema::NodePtr;
-
-namespace test {
-
-template <>
-void InitDictValues<bool>(int num_values, int dict_per_page, vector<bool>& values,
- vector<uint8_t>& buffer) {
- // No op for bool
-}
-
-template <typename Type>
-class TestFlatScanner : public ::testing::Test {
- public:
- typedef typename Type::c_type T;
-
- void InitScanner(const ColumnDescriptor* d) {
- std::unique_ptr<PageReader> pager(new test::MockPageReader(pages_));
- scanner_ = Scanner::Make(ColumnReader::Make(d, std::move(pager)));
- }
-
- void CheckResults(int batch_size, const ColumnDescriptor* d) {
- TypedScanner<Type>* scanner = reinterpret_cast<TypedScanner<Type>*>(scanner_.get());
- T val;
- bool is_null = false;
- int16_t def_level;
- int16_t rep_level;
- int j = 0;
- scanner->SetBatchSize(batch_size);
- for (int i = 0; i < num_levels_; i++) {
- ASSERT_TRUE(scanner->Next(&val, &def_level, &rep_level, &is_null)) << i << j;
- if (!is_null) {
- ASSERT_EQ(values_[j], val) << i << "V" << j;
- j++;
- }
- if (d->max_definition_level() > 0) {
- ASSERT_EQ(def_levels_[i], def_level) << i << "D" << j;
- }
- if (d->max_repetition_level() > 0) {
- ASSERT_EQ(rep_levels_[i], rep_level) << i << "R" << j;
- }
- }
- ASSERT_EQ(num_values_, j);
- ASSERT_FALSE(scanner->Next(&val, &def_level, &rep_level, &is_null));
- }
-
- void Clear() {
- pages_.clear();
- values_.clear();
- def_levels_.clear();
- rep_levels_.clear();
- }
-
- void Execute(int num_pages, int levels_per_page, int batch_size,
- const ColumnDescriptor* d, Encoding::type encoding) {
- num_values_ = MakePages<Type>(d, num_pages, levels_per_page, def_levels_, rep_levels_,
- values_, data_buffer_, pages_, encoding);
- num_levels_ = num_pages * levels_per_page;
- InitScanner(d);
- CheckResults(batch_size, d);
- Clear();
- }
-
- void InitDescriptors(std::shared_ptr<ColumnDescriptor>& d1,
- std::shared_ptr<ColumnDescriptor>& d2,
- std::shared_ptr<ColumnDescriptor>& d3, int length) {
- NodePtr type;
- type = schema::PrimitiveNode::Make("c1", Repetition::REQUIRED, Type::type_num,
- LogicalType::NONE, length);
- d1.reset(new ColumnDescriptor(type, 0, 0));
- type = schema::PrimitiveNode::Make("c2", Repetition::OPTIONAL, Type::type_num,
- LogicalType::NONE, length);
- d2.reset(new ColumnDescriptor(type, 4, 0));
- type = schema::PrimitiveNode::Make("c3", Repetition::REPEATED, Type::type_num,
- LogicalType::NONE, length);
- d3.reset(new ColumnDescriptor(type, 4, 2));
- }
-
- void ExecuteAll(int num_pages, int num_levels, int batch_size, int type_length,
- Encoding::type encoding = Encoding::PLAIN) {
- std::shared_ptr<ColumnDescriptor> d1;
- std::shared_ptr<ColumnDescriptor> d2;
- std::shared_ptr<ColumnDescriptor> d3;
- InitDescriptors(d1, d2, d3, type_length);
- // evaluate REQUIRED pages
- Execute(num_pages, num_levels, batch_size, d1.get(), encoding);
- // evaluate OPTIONAL pages
- Execute(num_pages, num_levels, batch_size, d2.get(), encoding);
- // evaluate REPEATED pages
- Execute(num_pages, num_levels, batch_size, d3.get(), encoding);
- }
-
- protected:
- int num_levels_;
- int num_values_;
- vector<shared_ptr<Page>> pages_;
- std::shared_ptr<Scanner> scanner_;
- vector<T> values_;
- vector<int16_t> def_levels_;
- vector<int16_t> rep_levels_;
- vector<uint8_t> data_buffer_; // For BA and FLBA
-};
-
-static int num_levels_per_page = 100;
-static int num_pages = 20;
-static int batch_size = 32;
-
-typedef ::testing::Types<Int32Type, Int64Type, Int96Type, FloatType, DoubleType,
- ByteArrayType>
- TestTypes;
-
-using TestBooleanFlatScanner = TestFlatScanner<BooleanType>;
-using TestFLBAFlatScanner = TestFlatScanner<FLBAType>;
-
-TYPED_TEST_CASE(TestFlatScanner, TestTypes);
-
-TYPED_TEST(TestFlatScanner, TestPlainScanner) {
- ASSERT_NO_FATAL_FAILURE(
- this->ExecuteAll(num_pages, num_levels_per_page, batch_size, 0, Encoding::PLAIN));
-}
-
-TYPED_TEST(TestFlatScanner, TestDictScanner) {
- ASSERT_NO_FATAL_FAILURE(this->ExecuteAll(num_pages, num_levels_per_page, batch_size, 0,
- Encoding::RLE_DICTIONARY));
-}
-
-TEST_F(TestBooleanFlatScanner, TestPlainScanner) {
- ASSERT_NO_FATAL_FAILURE(
- this->ExecuteAll(num_pages, num_levels_per_page, batch_size, 0));
-}
-
-TEST_F(TestFLBAFlatScanner, TestPlainScanner) {
- ASSERT_NO_FATAL_FAILURE(
- this->ExecuteAll(num_pages, num_levels_per_page, batch_size, FLBA_LENGTH));
-}
-
-TEST_F(TestFLBAFlatScanner, TestDictScanner) {
- ASSERT_NO_FATAL_FAILURE(this->ExecuteAll(num_pages, num_levels_per_page, batch_size,
- FLBA_LENGTH, Encoding::RLE_DICTIONARY));
-}
-
-TEST_F(TestFLBAFlatScanner, TestPlainDictScanner) {
- ASSERT_NO_FATAL_FAILURE(this->ExecuteAll(num_pages, num_levels_per_page, batch_size,
- FLBA_LENGTH, Encoding::PLAIN_DICTIONARY));
-}
-
-// PARQUET 502
-TEST_F(TestFLBAFlatScanner, TestSmallBatch) {
- NodePtr type =
- schema::PrimitiveNode::Make("c1", Repetition::REQUIRED, Type::FIXED_LEN_BYTE_ARRAY,
- LogicalType::DECIMAL, FLBA_LENGTH, 10, 2);
- const ColumnDescriptor d(type, 0, 0);
- num_values_ = MakePages<FLBAType>(&d, 1, 100, def_levels_, rep_levels_, values_,
- data_buffer_, pages_);
- num_levels_ = 1 * 100;
- InitScanner(&d);
- ASSERT_NO_FATAL_FAILURE(CheckResults(1, &d));
-}
-
-TEST_F(TestFLBAFlatScanner, TestDescriptorAPI) {
- NodePtr type =
- schema::PrimitiveNode::Make("c1", Repetition::OPTIONAL, Type::FIXED_LEN_BYTE_ARRAY,
- LogicalType::DECIMAL, FLBA_LENGTH, 10, 2);
- const ColumnDescriptor d(type, 4, 0);
- num_values_ = MakePages<FLBAType>(&d, 1, 100, def_levels_, rep_levels_, values_,
- data_buffer_, pages_);
- num_levels_ = 1 * 100;
- InitScanner(&d);
- TypedScanner<FLBAType>* scanner =
- reinterpret_cast<TypedScanner<FLBAType>*>(scanner_.get());
- ASSERT_EQ(10, scanner->descr()->type_precision());
- ASSERT_EQ(2, scanner->descr()->type_scale());
- ASSERT_EQ(FLBA_LENGTH, scanner->descr()->type_length());
-}
-
-TEST_F(TestFLBAFlatScanner, TestFLBAPrinterNext) {
- NodePtr type =
- schema::PrimitiveNode::Make("c1", Repetition::OPTIONAL, Type::FIXED_LEN_BYTE_ARRAY,
- LogicalType::DECIMAL, FLBA_LENGTH, 10, 2);
- const ColumnDescriptor d(type, 4, 0);
- num_values_ = MakePages<FLBAType>(&d, 1, 100, def_levels_, rep_levels_, values_,
- data_buffer_, pages_);
- num_levels_ = 1 * 100;
- InitScanner(&d);
- TypedScanner<FLBAType>* scanner =
- reinterpret_cast<TypedScanner<FLBAType>*>(scanner_.get());
- scanner->SetBatchSize(batch_size);
- std::stringstream ss_fail;
- for (int i = 0; i < num_levels_; i++) {
- std::stringstream ss;
- scanner->PrintNext(ss, 17);
- std::string result = ss.str();
- ASSERT_LE(17, result.size()) << i;
- }
- ASSERT_THROW(scanner->PrintNext(ss_fail, 17), ParquetException);
-}
-
-} // namespace test
-} // namespace parquet
diff --git a/src/parquet/column_scanner.cc b/src/parquet/column_scanner.cc
deleted file mode 100644
index 51c8773..0000000
--- a/src/parquet/column_scanner.cc
+++ /dev/null
@@ -1,91 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#include "parquet/column_scanner.h"
-
-#include <cstdint>
-#include <memory>
-
-#include "parquet/column_reader.h"
-#include "parquet/util/memory.h"
-
-using arrow::MemoryPool;
-
-namespace parquet {
-
-std::shared_ptr<Scanner> Scanner::Make(std::shared_ptr<ColumnReader> col_reader,
- int64_t batch_size, MemoryPool* pool) {
- switch (col_reader->type()) {
- case Type::BOOLEAN:
- return std::make_shared<BoolScanner>(col_reader, batch_size, pool);
- case Type::INT32:
- return std::make_shared<Int32Scanner>(col_reader, batch_size, pool);
- case Type::INT64:
- return std::make_shared<Int64Scanner>(col_reader, batch_size, pool);
- case Type::INT96:
- return std::make_shared<Int96Scanner>(col_reader, batch_size, pool);
- case Type::FLOAT:
- return std::make_shared<FloatScanner>(col_reader, batch_size, pool);
- case Type::DOUBLE:
- return std::make_shared<DoubleScanner>(col_reader, batch_size, pool);
- case Type::BYTE_ARRAY:
- return std::make_shared<ByteArrayScanner>(col_reader, batch_size, pool);
- case Type::FIXED_LEN_BYTE_ARRAY:
- return std::make_shared<FixedLenByteArrayScanner>(col_reader, batch_size, pool);
- default:
- ParquetException::NYI("type reader not implemented");
- }
- // Unreachable code, but supress compiler warning
- return std::shared_ptr<Scanner>(nullptr);
-}
-
-int64_t ScanAllValues(int32_t batch_size, int16_t* def_levels, int16_t* rep_levels,
- uint8_t* values, int64_t* values_buffered,
- parquet::ColumnReader* reader) {
- switch (reader->type()) {
- case parquet::Type::BOOLEAN:
- return ScanAll<parquet::BoolReader>(batch_size, def_levels, rep_levels, values,
- values_buffered, reader);
- case parquet::Type::INT32:
- return ScanAll<parquet::Int32Reader>(batch_size, def_levels, rep_levels, values,
- values_buffered, reader);
- case parquet::Type::INT64:
- return ScanAll<parquet::Int64Reader>(batch_size, def_levels, rep_levels, values,
- values_buffered, reader);
- case parquet::Type::INT96:
- return ScanAll<parquet::Int96Reader>(batch_size, def_levels, rep_levels, values,
- values_buffered, reader);
- case parquet::Type::FLOAT:
- return ScanAll<parquet::FloatReader>(batch_size, def_levels, rep_levels, values,
- values_buffered, reader);
- case parquet::Type::DOUBLE:
- return ScanAll<parquet::DoubleReader>(batch_size, def_levels, rep_levels, values,
- values_buffered, reader);
- case parquet::Type::BYTE_ARRAY:
- return ScanAll<parquet::ByteArrayReader>(batch_size, def_levels, rep_levels, values,
- values_buffered, reader);
- case parquet::Type::FIXED_LEN_BYTE_ARRAY:
- return ScanAll<parquet::FixedLenByteArrayReader>(batch_size, def_levels, rep_levels,
- values, values_buffered, reader);
- default:
- parquet::ParquetException::NYI("type reader not implemented");
- }
- // Unreachable code, but supress compiler warning
- return 0;
-}
-
-} // namespace parquet
diff --git a/src/parquet/column_scanner.h b/src/parquet/column_scanner.h
deleted file mode 100644
index 0a866ee..0000000
--- a/src/parquet/column_scanner.h
+++ /dev/null
@@ -1,257 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#ifndef PARQUET_COLUMN_SCANNER_H
-#define PARQUET_COLUMN_SCANNER_H
-
-#include <stdio.h>
-#include <cstdint>
-#include <memory>
-#include <ostream>
-#include <string>
-#include <vector>
-
-#include "parquet/column_reader.h"
-#include "parquet/exception.h"
-#include "parquet/schema.h"
-#include "parquet/types.h"
-#include "parquet/util/memory.h"
-#include "parquet/util/visibility.h"
-
-namespace parquet {
-
-static constexpr int64_t DEFAULT_SCANNER_BATCH_SIZE = 128;
-
-class PARQUET_EXPORT Scanner {
- public:
- explicit Scanner(std::shared_ptr<ColumnReader> reader,
- int64_t batch_size = DEFAULT_SCANNER_BATCH_SIZE,
- ::arrow::MemoryPool* pool = ::arrow::default_memory_pool())
- : batch_size_(batch_size),
- level_offset_(0),
- levels_buffered_(0),
- value_buffer_(AllocateBuffer(pool)),
- value_offset_(0),
- values_buffered_(0),
- reader_(reader) {
- def_levels_.resize(descr()->max_definition_level() > 0 ? batch_size_ : 0);
- rep_levels_.resize(descr()->max_repetition_level() > 0 ? batch_size_ : 0);
- }
-
- virtual ~Scanner() {}
-
- static std::shared_ptr<Scanner> Make(
- std::shared_ptr<ColumnReader> col_reader,
- int64_t batch_size = DEFAULT_SCANNER_BATCH_SIZE,
- ::arrow::MemoryPool* pool = ::arrow::default_memory_pool());
-
- virtual void PrintNext(std::ostream& out, int width) = 0;
-
- bool HasNext() { return level_offset_ < levels_buffered_ || reader_->HasNext(); }
-
- const ColumnDescriptor* descr() const { return reader_->descr(); }
-
- int64_t batch_size() const { return batch_size_; }
-
- void SetBatchSize(int64_t batch_size) { batch_size_ = batch_size; }
-
- protected:
- int64_t batch_size_;
-
- std::vector<int16_t> def_levels_;
- std::vector<int16_t> rep_levels_;
- int level_offset_;
- int levels_buffered_;
-
- std::shared_ptr<ResizableBuffer> value_buffer_;
- int value_offset_;
- int64_t values_buffered_;
-
- private:
- std::shared_ptr<ColumnReader> reader_;
-};
-
-template <typename DType>
-class PARQUET_EXPORT TypedScanner : public Scanner {
- public:
- typedef typename DType::c_type T;
-
- explicit TypedScanner(std::shared_ptr<ColumnReader> reader,
- int64_t batch_size = DEFAULT_SCANNER_BATCH_SIZE,
- ::arrow::MemoryPool* pool = ::arrow::default_memory_pool())
- : Scanner(reader, batch_size, pool) {
- typed_reader_ = static_cast<TypedColumnReader<DType>*>(reader.get());
- int value_byte_size = type_traits<DType::type_num>::value_byte_size;
- PARQUET_THROW_NOT_OK(value_buffer_->Resize(batch_size_ * value_byte_size));
- values_ = reinterpret_cast<T*>(value_buffer_->mutable_data());
- }
-
- virtual ~TypedScanner() {}
-
- bool NextLevels(int16_t* def_level, int16_t* rep_level) {
- if (level_offset_ == levels_buffered_) {
- levels_buffered_ = static_cast<int>(
- typed_reader_->ReadBatch(static_cast<int>(batch_size_), def_levels_.data(),
- rep_levels_.data(), values_, &values_buffered_));
-
- value_offset_ = 0;
- level_offset_ = 0;
- if (!levels_buffered_) {
- return false;
- }
- }
- *def_level = descr()->max_definition_level() > 0 ? def_levels_[level_offset_] : 0;
- *rep_level = descr()->max_repetition_level() > 0 ? rep_levels_[level_offset_] : 0;
- level_offset_++;
- return true;
- }
-
- bool Next(T* val, int16_t* def_level, int16_t* rep_level, bool* is_null) {
- if (level_offset_ == levels_buffered_) {
- if (!HasNext()) {
- // Out of data pages
- return false;
- }
- }
-
- NextLevels(def_level, rep_level);
- *is_null = *def_level < descr()->max_definition_level();
-
- if (*is_null) {
- return true;
- }
-
- if (value_offset_ == values_buffered_) {
- throw ParquetException("Value was non-null, but has not been buffered");
- }
- *val = values_[value_offset_++];
- return true;
- }
-
- // Returns true if there is a next value
- bool NextValue(T* val, bool* is_null) {
- if (level_offset_ == levels_buffered_) {
- if (!HasNext()) {
- // Out of data pages
- return false;
- }
- }
-
- // Out of values
- int16_t def_level = -1;
- int16_t rep_level = -1;
- NextLevels(&def_level, &rep_level);
- *is_null = def_level < descr()->max_definition_level();
-
- if (*is_null) {
- return true;
- }
-
- if (value_offset_ == values_buffered_) {
- throw ParquetException("Value was non-null, but has not been buffered");
- }
- *val = values_[value_offset_++];
- return true;
- }
-
- virtual void PrintNext(std::ostream& out, int width) {
- T val;
- bool is_null = false;
- char buffer[25];
-
- if (!NextValue(&val, &is_null)) {
- throw ParquetException("No more values buffered");
- }
-
- if (is_null) {
- std::string null_fmt = format_fwf<ByteArrayType>(width);
- snprintf(buffer, sizeof(buffer), null_fmt.c_str(), "NULL");
- } else {
- FormatValue(&val, buffer, sizeof(buffer), width);
- }
- out << buffer;
- }
-
- private:
- // The ownership of this object is expressed through the reader_ variable in the base
- TypedColumnReader<DType>* typed_reader_;
-
- inline void FormatValue(void* val, char* buffer, int bufsize, int width);
-
- T* values_;
-};
-
-template <typename DType>
-inline void TypedScanner<DType>::FormatValue(void* val, char* buffer, int bufsize,
- int width) {
- std::string fmt = format_fwf<DType>(width);
- snprintf(buffer, bufsize, fmt.c_str(), *reinterpret_cast<T*>(val));
-}
-
-template <>
-inline void TypedScanner<Int96Type>::FormatValue(void* val, char* buffer, int bufsize,
- int width) {
- std::string fmt = format_fwf<Int96Type>(width);
- std::string result = Int96ToString(*reinterpret_cast<Int96*>(val));
- snprintf(buffer, bufsize, fmt.c_str(), result.c_str());
-}
-
-template <>
-inline void TypedScanner<ByteArrayType>::FormatValue(void* val, char* buffer, int bufsize,
- int width) {
- std::string fmt = format_fwf<ByteArrayType>(width);
- std::string result = ByteArrayToString(*reinterpret_cast<ByteArray*>(val));
- snprintf(buffer, bufsize, fmt.c_str(), result.c_str());
-}
-
-template <>
-inline void TypedScanner<FLBAType>::FormatValue(void* val, char* buffer, int bufsize,
- int width) {
- std::string fmt = format_fwf<FLBAType>(width);
- std::string result = FixedLenByteArrayToString(
- *reinterpret_cast<FixedLenByteArray*>(val), descr()->type_length());
- snprintf(buffer, bufsize, fmt.c_str(), result.c_str());
-}
-
-typedef TypedScanner<BooleanType> BoolScanner;
-typedef TypedScanner<Int32Type> Int32Scanner;
-typedef TypedScanner<Int64Type> Int64Scanner;
-typedef TypedScanner<Int96Type> Int96Scanner;
-typedef TypedScanner<FloatType> FloatScanner;
-typedef TypedScanner<DoubleType> DoubleScanner;
-typedef TypedScanner<ByteArrayType> ByteArrayScanner;
-typedef TypedScanner<FLBAType> FixedLenByteArrayScanner;
-
-template <typename RType>
-int64_t ScanAll(int32_t batch_size, int16_t* def_levels, int16_t* rep_levels,
- uint8_t* values, int64_t* values_buffered,
- parquet::ColumnReader* reader) {
- typedef typename RType::T Type;
- auto typed_reader = static_cast<RType*>(reader);
- auto vals = reinterpret_cast<Type*>(&values[0]);
- return typed_reader->ReadBatch(batch_size, def_levels, rep_levels, vals,
- values_buffered);
-}
-
-int64_t PARQUET_EXPORT ScanAllValues(int32_t batch_size, int16_t* def_levels,
- int16_t* rep_levels, uint8_t* values,
- int64_t* values_buffered,
- parquet::ColumnReader* reader);
-
-} // namespace parquet
-
-#endif // PARQUET_COLUMN_SCANNER_H
diff --git a/src/parquet/column_writer-test.cc b/src/parquet/column_writer-test.cc
deleted file mode 100644
index e87d549..0000000
--- a/src/parquet/column_writer-test.cc
+++ /dev/null
@@ -1,807 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#include <gtest/gtest.h>
-
-#include "parquet/column_reader.h"
-#include "parquet/column_writer.h"
-#include "parquet/parquet_types.h"
-#include "parquet/test-specialization.h"
-#include "parquet/test-util.h"
-#include "parquet/types.h"
-#include "parquet/util/comparison.h"
-#include "parquet/util/memory.h"
-
-namespace parquet {
-
-using schema::NodePtr;
-using schema::PrimitiveNode;
-
-namespace test {
-
-// The default size used in most tests.
-const int SMALL_SIZE = 100;
-#ifdef PARQUET_VALGRIND
-// Larger size to test some corner cases, only used in some specific cases.
-const int LARGE_SIZE = 10000;
-// Very large size to test dictionary fallback.
-const int VERY_LARGE_SIZE = 40000;
-#else
-// Larger size to test some corner cases, only used in some specific cases.
-const int LARGE_SIZE = 100000;
-// Very large size to test dictionary fallback.
-const int VERY_LARGE_SIZE = 400000;
-#endif
-
-template <typename TestType>
-class TestPrimitiveWriter : public PrimitiveTypedTest<TestType> {
- public:
- typedef typename TestType::c_type T;
-
- void SetUp() {
- this->SetupValuesOut(SMALL_SIZE);
- writer_properties_ = default_writer_properties();
- definition_levels_out_.resize(SMALL_SIZE);
- repetition_levels_out_.resize(SMALL_SIZE);
-
- this->SetUpSchema(Repetition::REQUIRED);
-
- descr_ = this->schema_.Column(0);
- }
-
- Type::type type_num() { return TestType::type_num; }
-
- void BuildReader(int64_t num_rows,
- Compression::type compression = Compression::UNCOMPRESSED) {
- auto buffer = sink_->GetBuffer();
- std::unique_ptr<InMemoryInputStream> source(new InMemoryInputStream(buffer));
- std::unique_ptr<PageReader> page_reader =
- PageReader::Open(std::move(source), num_rows, compression);
- reader_.reset(new TypedColumnReader<TestType>(this->descr_, std::move(page_reader)));
- }
-
- std::shared_ptr<TypedColumnWriter<TestType>> BuildWriter(
- int64_t output_size = SMALL_SIZE,
- const ColumnProperties& column_properties = ColumnProperties()) {
- sink_.reset(new InMemoryOutputStream());
- WriterProperties::Builder wp_builder;
- if (column_properties.encoding() == Encoding::PLAIN_DICTIONARY ||
- column_properties.encoding() == Encoding::RLE_DICTIONARY) {
- wp_builder.enable_dictionary();
- } else {
- wp_builder.disable_dictionary();
- wp_builder.encoding(column_properties.encoding());
- }
- wp_builder.max_statistics_size(column_properties.max_statistics_size());
- writer_properties_ = wp_builder.build();
-
- metadata_ = ColumnChunkMetaDataBuilder::Make(
- writer_properties_, this->descr_, reinterpret_cast<uint8_t*>(&thrift_metadata_));
- std::unique_ptr<PageWriter> pager =
- PageWriter::Open(sink_.get(), column_properties.compression(), metadata_.get());
- std::shared_ptr<ColumnWriter> writer =
- ColumnWriter::Make(metadata_.get(), std::move(pager), writer_properties_.get());
- return std::static_pointer_cast<TypedColumnWriter<TestType>>(writer);
- }
-
- void ReadColumn(Compression::type compression = Compression::UNCOMPRESSED) {
- BuildReader(static_cast<int64_t>(this->values_out_.size()), compression);
- reader_->ReadBatch(static_cast<int>(this->values_out_.size()),
- definition_levels_out_.data(), repetition_levels_out_.data(),
- this->values_out_ptr_, &values_read_);
- this->SyncValuesOut();
- }
-
- void ReadColumnFully(Compression::type compression = Compression::UNCOMPRESSED);
-
- void TestRequiredWithEncoding(Encoding::type encoding) {
- return TestRequiredWithSettings(encoding, Compression::UNCOMPRESSED, false, false);
- }
-
- void TestRequiredWithSettings(Encoding::type encoding, Compression::type compression,
- bool enable_dictionary, bool enable_statistics,
- int64_t num_rows = SMALL_SIZE) {
- this->GenerateData(num_rows);
-
- this->WriteRequiredWithSettings(encoding, compression, enable_dictionary,
- enable_statistics, num_rows);
- ASSERT_NO_FATAL_FAILURE(this->ReadAndCompare(compression, num_rows));
-
- this->WriteRequiredWithSettingsSpaced(encoding, compression, enable_dictionary,
- enable_statistics, num_rows);
- ASSERT_NO_FATAL_FAILURE(this->ReadAndCompare(compression, num_rows));
- }
-
- void WriteRequiredWithSettings(Encoding::type encoding, Compression::type compression,
- bool enable_dictionary, bool enable_statistics,
- int64_t num_rows) {
- ColumnProperties column_properties(encoding, compression, enable_dictionary,
- enable_statistics);
- std::shared_ptr<TypedColumnWriter<TestType>> writer =
- this->BuildWriter(num_rows, column_properties);
- writer->WriteBatch(this->values_.size(), nullptr, nullptr, this->values_ptr_);
- // The behaviour should be independent from the number of Close() calls
- writer->Close();
- writer->Close();
- }
-
- void WriteRequiredWithSettingsSpaced(Encoding::type encoding,
- Compression::type compression,
- bool enable_dictionary, bool enable_statistics,
- int64_t num_rows) {
- std::vector<uint8_t> valid_bits(
- BitUtil::BytesForBits(static_cast<uint32_t>(this->values_.size())) + 1, 255);
- ColumnProperties column_properties(encoding, compression, enable_dictionary,
- enable_statistics);
- std::shared_ptr<TypedColumnWriter<TestType>> writer =
- this->BuildWriter(num_rows, column_properties);
- writer->WriteBatchSpaced(this->values_.size(), nullptr, nullptr, valid_bits.data(), 0,
- this->values_ptr_);
- // The behaviour should be independent from the number of Close() calls
- writer->Close();
- writer->Close();
- }
-
- void ReadAndCompare(Compression::type compression, int64_t num_rows) {
- this->SetupValuesOut(num_rows);
- this->ReadColumnFully(compression);
- std::shared_ptr<CompareDefault<TestType>> compare;
- compare = std::static_pointer_cast<CompareDefault<TestType>>(
- Comparator::Make(this->descr_));
- for (size_t i = 0; i < this->values_.size(); i++) {
- if ((*compare)(this->values_[i], this->values_out_[i]) ||
- (*compare)(this->values_out_[i], this->values_[i])) {
- std::cout << "Failed at " << i << std::endl;
- }
- ASSERT_FALSE((*compare)(this->values_[i], this->values_out_[i]));
- ASSERT_FALSE((*compare)(this->values_out_[i], this->values_[i]));
- }
- ASSERT_EQ(this->values_, this->values_out_);
- }
-
- int64_t metadata_num_values() {
- // Metadata accessor must be created lazily.
- // This is because the ColumnChunkMetaData semantics dictate the metadata object is
- // complete (no changes to the metadata buffer can be made after instantiation)
- auto metadata_accessor = ColumnChunkMetaData::Make(
- reinterpret_cast<const uint8_t*>(&thrift_metadata_), this->descr_);
- return metadata_accessor->num_values();
- }
-
- bool metadata_is_stats_set() {
- // Metadata accessor must be created lazily.
- // This is because the ColumnChunkMetaData semantics dictate the metadata object is
- // complete (no changes to the metadata buffer can be made after instantiation)
- ApplicationVersion app_version(this->writer_properties_->created_by());
- auto metadata_accessor = ColumnChunkMetaData::Make(
- reinterpret_cast<const uint8_t*>(&thrift_metadata_), this->descr_, &app_version);
- return metadata_accessor->is_stats_set();
- }
-
- std::vector<Encoding::type> metadata_encodings() {
- // Metadata accessor must be created lazily.
- // This is because the ColumnChunkMetaData semantics dictate the metadata object is
- // complete (no changes to the metadata buffer can be made after instantiation)
- auto metadata_accessor = ColumnChunkMetaData::Make(
- reinterpret_cast<const uint8_t*>(&thrift_metadata_), this->descr_);
- return metadata_accessor->encodings();
- }
-
- protected:
- int64_t values_read_;
- // Keep the reader alive as for ByteArray the lifetime of the ByteArray
- // content is bound to the reader.
- std::unique_ptr<TypedColumnReader<TestType>> reader_;
-
- std::vector<int16_t> definition_levels_out_;
- std::vector<int16_t> repetition_levels_out_;
-
- const ColumnDescriptor* descr_;
-
- private:
- format::ColumnChunk thrift_metadata_;
- std::unique_ptr<ColumnChunkMetaDataBuilder> metadata_;
- std::unique_ptr<InMemoryOutputStream> sink_;
- std::shared_ptr<WriterProperties> writer_properties_;
- std::vector<std::vector<uint8_t>> data_buffer_;
-};
-
-template <typename TestType>
-void TestPrimitiveWriter<TestType>::ReadColumnFully(Compression::type compression) {
- int64_t total_values = static_cast<int64_t>(this->values_out_.size());
- BuildReader(total_values, compression);
- values_read_ = 0;
- while (values_read_ < total_values) {
- int64_t values_read_recently = 0;
- reader_->ReadBatch(
- static_cast<int>(this->values_out_.size()) - static_cast<int>(values_read_),
- definition_levels_out_.data() + values_read_,
- repetition_levels_out_.data() + values_read_,
- this->values_out_ptr_ + values_read_, &values_read_recently);
- values_read_ += values_read_recently;
- }
- this->SyncValuesOut();
-}
-
-template <>
-void TestPrimitiveWriter<Int96Type>::ReadAndCompare(Compression::type compression,
- int64_t num_rows) {
- this->SetupValuesOut(num_rows);
- this->ReadColumnFully(compression);
- std::shared_ptr<CompareDefault<Int96Type>> compare;
- compare = std::make_shared<CompareDefaultInt96>();
- for (size_t i = 0; i < this->values_.size(); i++) {
- if ((*compare)(this->values_[i], this->values_out_[i]) ||
- (*compare)(this->values_out_[i], this->values_[i])) {
- std::cout << "Failed at " << i << std::endl;
- }
- ASSERT_FALSE((*compare)(this->values_[i], this->values_out_[i]));
- ASSERT_FALSE((*compare)(this->values_out_[i], this->values_[i]));
- }
- ASSERT_EQ(this->values_, this->values_out_);
-}
-
-template <>
-void TestPrimitiveWriter<FLBAType>::ReadColumnFully(Compression::type compression) {
- int64_t total_values = static_cast<int64_t>(this->values_out_.size());
- BuildReader(total_values, compression);
- this->data_buffer_.clear();
-
- values_read_ = 0;
- while (values_read_ < total_values) {
- int64_t values_read_recently = 0;
- reader_->ReadBatch(
- static_cast<int>(this->values_out_.size()) - static_cast<int>(values_read_),
- definition_levels_out_.data() + values_read_,
- repetition_levels_out_.data() + values_read_,
- this->values_out_ptr_ + values_read_, &values_read_recently);
-
- // Copy contents of the pointers
- std::vector<uint8_t> data(values_read_recently * this->descr_->type_length());
- uint8_t* data_ptr = data.data();
- for (int64_t i = 0; i < values_read_recently; i++) {
- memcpy(data_ptr + this->descr_->type_length() * i,
- this->values_out_[i + values_read_].ptr, this->descr_->type_length());
- this->values_out_[i + values_read_].ptr =
- data_ptr + this->descr_->type_length() * i;
- }
- data_buffer_.emplace_back(std::move(data));
-
- values_read_ += values_read_recently;
- }
- this->SyncValuesOut();
-}
-
-typedef ::testing::Types<Int32Type, Int64Type, Int96Type, FloatType, DoubleType,
- BooleanType, ByteArrayType, FLBAType>
- TestTypes;
-
-TYPED_TEST_CASE(TestPrimitiveWriter, TestTypes);
-
-using TestNullValuesWriter = TestPrimitiveWriter<Int32Type>;
-
-TYPED_TEST(TestPrimitiveWriter, RequiredPlain) {
- this->TestRequiredWithEncoding(Encoding::PLAIN);
-}
-
-TYPED_TEST(TestPrimitiveWriter, RequiredDictionary) {
- this->TestRequiredWithEncoding(Encoding::PLAIN_DICTIONARY);
-}
-
-/*
-TYPED_TEST(TestPrimitiveWriter, RequiredRLE) {
- this->TestRequiredWithEncoding(Encoding::RLE);
-}
-
-TYPED_TEST(TestPrimitiveWriter, RequiredBitPacked) {
- this->TestRequiredWithEncoding(Encoding::BIT_PACKED);
-}
-
-TYPED_TEST(TestPrimitiveWriter, RequiredDeltaBinaryPacked) {
- this->TestRequiredWithEncoding(Encoding::DELTA_BINARY_PACKED);
-}
-
-TYPED_TEST(TestPrimitiveWriter, RequiredDeltaLengthByteArray) {
- this->TestRequiredWithEncoding(Encoding::DELTA_LENGTH_BYTE_ARRAY);
-}
-
-TYPED_TEST(TestPrimitiveWriter, RequiredDeltaByteArray) {
- this->TestRequiredWithEncoding(Encoding::DELTA_BYTE_ARRAY);
-}
-
-TYPED_TEST(TestPrimitiveWriter, RequiredRLEDictionary) {
- this->TestRequiredWithEncoding(Encoding::RLE_DICTIONARY);
-}
-*/
-
-TYPED_TEST(TestPrimitiveWriter, RequiredPlainWithSnappyCompression) {
- this->TestRequiredWithSettings(Encoding::PLAIN, Compression::SNAPPY, false, false,
- LARGE_SIZE);
-}
-
-TYPED_TEST(TestPrimitiveWriter, RequiredPlainWithBrotliCompression) {
- this->TestRequiredWithSettings(Encoding::PLAIN, Compression::BROTLI, false, false,
- LARGE_SIZE);
-}
-
-TYPED_TEST(TestPrimitiveWriter, RequiredPlainWithGzipCompression) {
- this->TestRequiredWithSettings(Encoding::PLAIN, Compression::GZIP, false, false,
- LARGE_SIZE);
-}
-
-TYPED_TEST(TestPrimitiveWriter, RequiredPlainWithLz4Compression) {
- this->TestRequiredWithSettings(Encoding::PLAIN, Compression::LZ4, false, false,
- LARGE_SIZE);
-}
-
-TYPED_TEST(TestPrimitiveWriter, RequiredPlainWithZstdCompression) {
- this->TestRequiredWithSettings(Encoding::PLAIN, Compression::ZSTD, false, false,
- LARGE_SIZE);
-}
-
-TYPED_TEST(TestPrimitiveWriter, RequiredPlainWithStats) {
- this->TestRequiredWithSettings(Encoding::PLAIN, Compression::UNCOMPRESSED, false, true,
- LARGE_SIZE);
-}
-
-TYPED_TEST(TestPrimitiveWriter, RequiredPlainWithStatsAndSnappyCompression) {
- this->TestRequiredWithSettings(Encoding::PLAIN, Compression::SNAPPY, false, true,
- LARGE_SIZE);
-}
-
-TYPED_TEST(TestPrimitiveWriter, RequiredPlainWithStatsAndBrotliCompression) {
- this->TestRequiredWithSettings(Encoding::PLAIN, Compression::BROTLI, false, true,
- LARGE_SIZE);
-}
-
-TYPED_TEST(TestPrimitiveWriter, RequiredPlainWithStatsAndGzipCompression) {
- this->TestRequiredWithSettings(Encoding::PLAIN, Compression::GZIP, false, true,
- LARGE_SIZE);
-}
-
-TYPED_TEST(TestPrimitiveWriter, RequiredPlainWithStatsAndLz4Compression) {
- this->TestRequiredWithSettings(Encoding::PLAIN, Compression::LZ4, false, true,
- LARGE_SIZE);
-}
-
-TYPED_TEST(TestPrimitiveWriter, RequiredPlainWithStatsAndZstdCompression) {
- this->TestRequiredWithSettings(Encoding::PLAIN, Compression::ZSTD, false, true,
- LARGE_SIZE);
-}
-
-TYPED_TEST(TestPrimitiveWriter, Optional) {
- // Optional and non-repeated, with definition levels
- // but no repetition levels
- this->SetUpSchema(Repetition::OPTIONAL);
-
- this->GenerateData(SMALL_SIZE);
- std::vector<int16_t> definition_levels(SMALL_SIZE, 1);
- definition_levels[1] = 0;
-
- auto writer = this->BuildWriter();
- writer->WriteBatch(this->values_.size(), definition_levels.data(), nullptr,
- this->values_ptr_);
- writer->Close();
-
- // PARQUET-703
- ASSERT_EQ(100, this->metadata_num_values());
-
- this->ReadColumn();
- ASSERT_EQ(99, this->values_read_);
- this->values_out_.resize(99);
- this->values_.resize(99);
- ASSERT_EQ(this->values_, this->values_out_);
-}
-
-TYPED_TEST(TestPrimitiveWriter, OptionalSpaced) {
- // Optional and non-repeated, with definition levels
- // but no repetition levels
- this->SetUpSchema(Repetition::OPTIONAL);
-
- this->GenerateData(SMALL_SIZE);
- std::vector<int16_t> definition_levels(SMALL_SIZE, 1);
- std::vector<uint8_t> valid_bits(::arrow::BitUtil::BytesForBits(SMALL_SIZE), 255);
-
- definition_levels[SMALL_SIZE - 1] = 0;
- ::arrow::BitUtil::ClearBit(valid_bits.data(), SMALL_SIZE - 1);
- definition_levels[1] = 0;
- ::arrow::BitUtil::ClearBit(valid_bits.data(), 1);
-
- auto writer = this->BuildWriter();
- writer->WriteBatchSpaced(this->values_.size(), definition_levels.data(), nullptr,
- valid_bits.data(), 0, this->values_ptr_);
- writer->Close();
-
- // PARQUET-703
- ASSERT_EQ(100, this->metadata_num_values());
-
- this->ReadColumn();
- ASSERT_EQ(98, this->values_read_);
- this->values_out_.resize(98);
- this->values_.resize(99);
- this->values_.erase(this->values_.begin() + 1);
- ASSERT_EQ(this->values_, this->values_out_);
-}
-
-TYPED_TEST(TestPrimitiveWriter, Repeated) {
- // Optional and repeated, so definition and repetition levels
- this->SetUpSchema(Repetition::REPEATED);
-
- this->GenerateData(SMALL_SIZE);
- std::vector<int16_t> definition_levels(SMALL_SIZE, 1);
- definition_levels[1] = 0;
- std::vector<int16_t> repetition_levels(SMALL_SIZE, 0);
-
- auto writer = this->BuildWriter();
- writer->WriteBatch(this->values_.size(), definition_levels.data(),
- repetition_levels.data(), this->values_ptr_);
- writer->Close();
-
- this->ReadColumn();
- ASSERT_EQ(SMALL_SIZE - 1, this->values_read_);
- this->values_out_.resize(SMALL_SIZE - 1);
- this->values_.resize(SMALL_SIZE - 1);
- ASSERT_EQ(this->values_, this->values_out_);
-}
-
-TYPED_TEST(TestPrimitiveWriter, RequiredLargeChunk) {
- this->GenerateData(LARGE_SIZE);
-
- // Test case 1: required and non-repeated, so no definition or repetition levels
- auto writer = this->BuildWriter(LARGE_SIZE);
- writer->WriteBatch(this->values_.size(), nullptr, nullptr, this->values_ptr_);
- writer->Close();
-
- // Just read the first SMALL_SIZE rows to ensure we could read it back in
- this->ReadColumn();
- ASSERT_EQ(SMALL_SIZE, this->values_read_);
- this->values_.resize(SMALL_SIZE);
- ASSERT_EQ(this->values_, this->values_out_);
-}
-
-// Test case for dictionary fallback encoding
-TYPED_TEST(TestPrimitiveWriter, RequiredVeryLargeChunk) {
- this->GenerateData(VERY_LARGE_SIZE);
-
- auto writer = this->BuildWriter(VERY_LARGE_SIZE, Encoding::PLAIN_DICTIONARY);
- writer->WriteBatch(this->values_.size(), nullptr, nullptr, this->values_ptr_);
- writer->Close();
-
- // Read all rows so we are sure that also the non-dictionary pages are read correctly
- this->SetupValuesOut(VERY_LARGE_SIZE);
- this->ReadColumnFully();
- ASSERT_EQ(VERY_LARGE_SIZE, this->values_read_);
- this->values_.resize(VERY_LARGE_SIZE);
- ASSERT_EQ(this->values_, this->values_out_);
- std::vector<Encoding::type> encodings = this->metadata_encodings();
- // There are 3 encodings (RLE, PLAIN_DICTIONARY, PLAIN) in a fallback case
- // Dictionary encoding is not allowed for boolean type
- // There are 2 encodings (RLE, PLAIN) in a non dictionary encoding case
- if (this->type_num() != Type::BOOLEAN) {
- ASSERT_EQ(Encoding::PLAIN_DICTIONARY, encodings[0]);
- ASSERT_EQ(Encoding::PLAIN, encodings[1]);
- ASSERT_EQ(Encoding::RLE, encodings[2]);
- } else {
- ASSERT_EQ(Encoding::PLAIN, encodings[0]);
- ASSERT_EQ(Encoding::RLE, encodings[1]);
- }
-}
-
-// PARQUET-719
-// Test case for NULL values
-TEST_F(TestNullValuesWriter, OptionalNullValueChunk) {
- this->SetUpSchema(Repetition::OPTIONAL);
-
- this->GenerateData(LARGE_SIZE);
-
- std::vector<int16_t> definition_levels(LARGE_SIZE, 0);
- std::vector<int16_t> repetition_levels(LARGE_SIZE, 0);
-
- auto writer = this->BuildWriter(LARGE_SIZE);
- // All values being written are NULL
- writer->WriteBatch(this->values_.size(), definition_levels.data(),
- repetition_levels.data(), nullptr);
- writer->Close();
-
- // Just read the first SMALL_SIZE rows to ensure we could read it back in
- this->ReadColumn();
- ASSERT_EQ(0, this->values_read_);
-}
-
-// PARQUET-764
-// Correct bitpacking for boolean write at non-byte boundaries
-using TestBooleanValuesWriter = TestPrimitiveWriter<BooleanType>;
-TEST_F(TestBooleanValuesWriter, AlternateBooleanValues) {
- this->SetUpSchema(Repetition::REQUIRED);
- auto writer = this->BuildWriter();
- for (int i = 0; i < SMALL_SIZE; i++) {
- bool value = (i % 2 == 0) ? true : false;
- writer->WriteBatch(1, nullptr, nullptr, &value);
- }
- writer->Close();
- this->ReadColumn();
- for (int i = 0; i < SMALL_SIZE; i++) {
- ASSERT_EQ((i % 2 == 0) ? true : false, this->values_out_[i]) << i;
- }
-}
-
-// PARQUET-979
-// Prevent writing large stats
-using TestByteArrayValuesWriter = TestPrimitiveWriter<ByteArrayType>;
-TEST_F(TestByteArrayValuesWriter, OmitStats) {
- int min_len = 1024 * 4;
- int max_len = 1024 * 8;
- this->SetUpSchema(Repetition::REQUIRED);
- auto writer = this->BuildWriter();
-
- values_.resize(SMALL_SIZE);
- InitWideByteArrayValues(SMALL_SIZE, this->values_, this->buffer_, min_len, max_len);
- writer->WriteBatch(SMALL_SIZE, nullptr, nullptr, this->values_.data());
- writer->Close();
-
- ASSERT_FALSE(this->metadata_is_stats_set());
-}
-
-TEST_F(TestByteArrayValuesWriter, LimitStats) {
- int min_len = 1024 * 4;
- int max_len = 1024 * 8;
- this->SetUpSchema(Repetition::REQUIRED);
- ColumnProperties column_properties;
- column_properties.set_max_statistics_size(static_cast<size_t>(max_len));
- auto writer = this->BuildWriter(SMALL_SIZE, column_properties);
-
- values_.resize(SMALL_SIZE);
- InitWideByteArrayValues(SMALL_SIZE, this->values_, this->buffer_, min_len, max_len);
- writer->WriteBatch(SMALL_SIZE, nullptr, nullptr, this->values_.data());
- writer->Close();
-
- ASSERT_TRUE(this->metadata_is_stats_set());
-}
-
-TEST_F(TestByteArrayValuesWriter, CheckDefaultStats) {
- this->SetUpSchema(Repetition::REQUIRED);
- auto writer = this->BuildWriter();
- this->GenerateData(SMALL_SIZE);
-
- writer->WriteBatch(SMALL_SIZE, nullptr, nullptr, this->values_ptr_);
- writer->Close();
-
- ASSERT_TRUE(this->metadata_is_stats_set());
-}
-
-void GenerateLevels(int min_repeat_factor, int max_repeat_factor, int max_level,
- std::vector<int16_t>& input_levels) {
- // for each repetition count upto max_repeat_factor
- for (int repeat = min_repeat_factor; repeat <= max_repeat_factor; repeat++) {
- // repeat count increases by a factor of 2 for every iteration
- int repeat_count = (1 << repeat);
- // generate levels for repetition count upto the maximum level
- int16_t value = 0;
- int bwidth = 0;
- while (value <= max_level) {
- for (int i = 0; i < repeat_count; i++) {
- input_levels.push_back(value);
- }
- value = static_cast<int16_t>((2 << bwidth) - 1);
- bwidth++;
- }
- }
-}
-
-void EncodeLevels(Encoding::type encoding, int16_t max_level, int num_levels,
- const int16_t* input_levels, std::vector<uint8_t>& bytes) {
- LevelEncoder encoder;
- int levels_count = 0;
- bytes.resize(2 * num_levels);
- ASSERT_EQ(2 * num_levels, static_cast<int>(bytes.size()));
- // encode levels
- if (encoding == Encoding::RLE) {
- // leave space to write the rle length value
- encoder.Init(encoding, max_level, num_levels, bytes.data() + sizeof(int32_t),
- static_cast<int>(bytes.size()));
-
- levels_count = encoder.Encode(num_levels, input_levels);
- (reinterpret_cast<int32_t*>(bytes.data()))[0] = encoder.len();
- } else {
- encoder.Init(encoding, max_level, num_levels, bytes.data(),
- static_cast<int>(bytes.size()));
- levels_count = encoder.Encode(num_levels, input_levels);
- }
- ASSERT_EQ(num_levels, levels_count);
-}
-
-void VerifyDecodingLevels(Encoding::type encoding, int16_t max_level,
- std::vector<int16_t>& input_levels,
- std::vector<uint8_t>& bytes) {
- LevelDecoder decoder;
- int levels_count = 0;
- std::vector<int16_t> output_levels;
- int num_levels = static_cast<int>(input_levels.size());
-
- output_levels.resize(num_levels);
- ASSERT_EQ(num_levels, static_cast<int>(output_levels.size()));
-
- // Decode levels and test with multiple decode calls
- decoder.SetData(encoding, max_level, num_levels, bytes.data());
- int decode_count = 4;
- int num_inner_levels = num_levels / decode_count;
- // Try multiple decoding on a single SetData call
- for (int ct = 0; ct < decode_count; ct++) {
- int offset = ct * num_inner_levels;
- levels_count = decoder.Decode(num_inner_levels, output_levels.data());
- ASSERT_EQ(num_inner_levels, levels_count);
- for (int i = 0; i < num_inner_levels; i++) {
- EXPECT_EQ(input_levels[i + offset], output_levels[i]);
- }
- }
- // check the remaining levels
- int num_levels_completed = decode_count * (num_levels / decode_count);
- int num_remaining_levels = num_levels - num_levels_completed;
- if (num_remaining_levels > 0) {
- levels_count = decoder.Decode(num_remaining_levels, output_levels.data());
- ASSERT_EQ(num_remaining_levels, levels_count);
- for (int i = 0; i < num_remaining_levels; i++) {
- EXPECT_EQ(input_levels[i + num_levels_completed], output_levels[i]);
- }
- }
- // Test zero Decode values
- ASSERT_EQ(0, decoder.Decode(1, output_levels.data()));
-}
-
-void VerifyDecodingMultipleSetData(Encoding::type encoding, int16_t max_level,
- std::vector<int16_t>& input_levels,
- std::vector<std::vector<uint8_t>>& bytes) {
- LevelDecoder decoder;
- int levels_count = 0;
- std::vector<int16_t> output_levels;
-
- // Decode levels and test with multiple SetData calls
- int setdata_count = static_cast<int>(bytes.size());
- int num_levels = static_cast<int>(input_levels.size()) / setdata_count;
- output_levels.resize(num_levels);
- // Try multiple SetData
- for (int ct = 0; ct < setdata_count; ct++) {
- int offset = ct * num_levels;
- ASSERT_EQ(num_levels, static_cast<int>(output_levels.size()));
- decoder.SetData(encoding, max_level, num_levels, bytes[ct].data());
- levels_count = decoder.Decode(num_levels, output_levels.data());
- ASSERT_EQ(num_levels, levels_count);
- for (int i = 0; i < num_levels; i++) {
- EXPECT_EQ(input_levels[i + offset], output_levels[i]);
- }
- }
-}
-
-// Test levels with maximum bit-width from 1 to 8
-// increase the repetition count for each iteration by a factor of 2
-TEST(TestLevels, TestLevelsDecodeMultipleBitWidth) {
- int min_repeat_factor = 0;
- int max_repeat_factor = 7; // 128
- int max_bit_width = 8;
- std::vector<int16_t> input_levels;
- std::vector<uint8_t> bytes;
- Encoding::type encodings[2] = {Encoding::RLE, Encoding::BIT_PACKED};
-
- // for each encoding
- for (int encode = 0; encode < 2; encode++) {
- Encoding::type encoding = encodings[encode];
- // BIT_PACKED requires a sequence of atleast 8
- if (encoding == Encoding::BIT_PACKED) min_repeat_factor = 3;
- // for each maximum bit-width
- for (int bit_width = 1; bit_width <= max_bit_width; bit_width++) {
- // find the maximum level for the current bit_width
- int16_t max_level = static_cast<int16_t>((1 << bit_width) - 1);
- // Generate levels
- GenerateLevels(min_repeat_factor, max_repeat_factor, max_level, input_levels);
- ASSERT_NO_FATAL_FAILURE(EncodeLevels(encoding, max_level,
- static_cast<int>(input_levels.size()),
- input_levels.data(), bytes));
- ASSERT_NO_FATAL_FAILURE(
- VerifyDecodingLevels(encoding, max_level, input_levels, bytes));
- input_levels.clear();
- }
- }
-}
-
-// Test multiple decoder SetData calls
-TEST(TestLevels, TestLevelsDecodeMultipleSetData) {
- int min_repeat_factor = 3;
- int max_repeat_factor = 7; // 128
- int bit_width = 8;
- int16_t max_level = static_cast<int16_t>((1 << bit_width) - 1);
- std::vector<int16_t> input_levels;
- std::vector<std::vector<uint8_t>> bytes;
- Encoding::type encodings[2] = {Encoding::RLE, Encoding::BIT_PACKED};
- GenerateLevels(min_repeat_factor, max_repeat_factor, max_level, input_levels);
- int num_levels = static_cast<int>(input_levels.size());
- int setdata_factor = 8;
- int split_level_size = num_levels / setdata_factor;
- bytes.resize(setdata_factor);
-
- // for each encoding
- for (int encode = 0; encode < 2; encode++) {
- Encoding::type encoding = encodings[encode];
- for (int rf = 0; rf < setdata_factor; rf++) {
- int offset = rf * split_level_size;
- ASSERT_NO_FATAL_FAILURE(EncodeLevels(
- encoding, max_level, split_level_size,
- reinterpret_cast<int16_t*>(input_levels.data()) + offset, bytes[rf]));
- }
- ASSERT_NO_FATAL_FAILURE(
- VerifyDecodingMultipleSetData(encoding, max_level, input_levels, bytes));
- }
-}
-
-TEST(TestLevelEncoder, MinimumBufferSize) {
- // PARQUET-676, PARQUET-698
- const int kNumToEncode = 1024;
-
- std::vector<int16_t> levels;
- for (int i = 0; i < kNumToEncode; ++i) {
- if (i % 9 == 0) {
- levels.push_back(0);
- } else {
- levels.push_back(1);
- }
- }
-
- std::vector<uint8_t> output(
- LevelEncoder::MaxBufferSize(Encoding::RLE, 1, kNumToEncode));
-
- LevelEncoder encoder;
- encoder.Init(Encoding::RLE, 1, kNumToEncode, output.data(),
- static_cast<int>(output.size()));
- int encode_count = encoder.Encode(kNumToEncode, levels.data());
-
- ASSERT_EQ(kNumToEncode, encode_count);
-}
-
-TEST(TestLevelEncoder, MinimumBufferSize2) {
- // PARQUET-708
- // Test the worst case for bit_width=2 consisting of
- // LiteralRun(size=8)
- // RepeatedRun(size=8)
- // LiteralRun(size=8)
- // ...
- const int kNumToEncode = 1024;
-
- std::vector<int16_t> levels;
- for (int i = 0; i < kNumToEncode; ++i) {
- // This forces a literal run of 00000001
- // followed by eight 1s
- if ((i % 16) < 7) {
- levels.push_back(0);
- } else {
- levels.push_back(1);
- }
- }
-
- for (int16_t bit_width = 1; bit_width <= 8; bit_width++) {
- std::vector<uint8_t> output(
- LevelEncoder::MaxBufferSize(Encoding::RLE, bit_width, kNumToEncode));
-
- LevelEncoder encoder;
- encoder.Init(Encoding::RLE, bit_width, kNumToEncode, output.data(),
- static_cast<int>(output.size()));
- int encode_count = encoder.Encode(kNumToEncode, levels.data());
-
- ASSERT_EQ(kNumToEncode, encode_count);
- }
-}
-
-} // namespace test
-} // namespace parquet
diff --git a/src/parquet/column_writer.cc b/src/parquet/column_writer.cc
deleted file mode 100644
index a65bda8..0000000
--- a/src/parquet/column_writer.cc
+++ /dev/null
@@ -1,867 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#include "parquet/column_writer.h"
-
-#include <cstdint>
-#include <memory>
-
-#include "arrow/util/bit-util.h"
-#include "arrow/util/compression.h"
-#include "arrow/util/rle-encoding.h"
-
-#include "parquet/encoding-internal.h"
-#include "parquet/properties.h"
-#include "parquet/statistics.h"
-#include "parquet/thrift.h"
-#include "parquet/util/logging.h"
-#include "parquet/util/memory.h"
-
-namespace parquet {
-
-using BitWriter = ::arrow::BitWriter;
-using RleEncoder = ::arrow::RleEncoder;
-
-LevelEncoder::LevelEncoder() {}
-LevelEncoder::~LevelEncoder() {}
-
-void LevelEncoder::Init(Encoding::type encoding, int16_t max_level,
- int num_buffered_values, uint8_t* data, int data_size) {
- bit_width_ = BitUtil::Log2(max_level + 1);
- encoding_ = encoding;
- switch (encoding) {
- case Encoding::RLE: {
- rle_encoder_.reset(new RleEncoder(data, data_size, bit_width_));
- break;
- }
- case Encoding::BIT_PACKED: {
- int num_bytes =
- static_cast<int>(BitUtil::BytesForBits(num_buffered_values * bit_width_));
- bit_packed_encoder_.reset(new BitWriter(data, num_bytes));
- break;
- }
- default:
- throw ParquetException("Unknown encoding type for levels.");
- }
-}
-
-int LevelEncoder::MaxBufferSize(Encoding::type encoding, int16_t max_level,
- int num_buffered_values) {
- int bit_width = BitUtil::Log2(max_level + 1);
- int num_bytes = 0;
- switch (encoding) {
- case Encoding::RLE: {
- // TODO: Due to the way we currently check if the buffer is full enough,
- // we need to have MinBufferSize as head room.
- num_bytes = RleEncoder::MaxBufferSize(bit_width, num_buffered_values) +
- RleEncoder::MinBufferSize(bit_width);
- break;
- }
- case Encoding::BIT_PACKED: {
- num_bytes =
- static_cast<int>(BitUtil::BytesForBits(num_buffered_values * bit_width));
- break;
- }
- default:
- throw ParquetException("Unknown encoding type for levels.");
- }
- return num_bytes;
-}
-
-int LevelEncoder::Encode(int batch_size, const int16_t* levels) {
- int num_encoded = 0;
- if (!rle_encoder_ && !bit_packed_encoder_) {
- throw ParquetException("Level encoders are not initialized.");
- }
-
- if (encoding_ == Encoding::RLE) {
- for (int i = 0; i < batch_size; ++i) {
- if (!rle_encoder_->Put(*(levels + i))) {
- break;
- }
- ++num_encoded;
- }
- rle_encoder_->Flush();
- rle_length_ = rle_encoder_->len();
- } else {
- for (int i = 0; i < batch_size; ++i) {
- if (!bit_packed_encoder_->PutValue(*(levels + i), bit_width_)) {
- break;
- }
- ++num_encoded;
- }
- bit_packed_encoder_->Flush();
- }
- return num_encoded;
-}
-
-// ----------------------------------------------------------------------
-// PageWriter implementation
-
-static format::Statistics ToThrift(const EncodedStatistics& row_group_statistics) {
- format::Statistics statistics;
- if (row_group_statistics.has_min) statistics.__set_min(row_group_statistics.min());
- if (row_group_statistics.has_max) statistics.__set_max(row_group_statistics.max());
- if (row_group_statistics.has_null_count)
- statistics.__set_null_count(row_group_statistics.null_count);
- if (row_group_statistics.has_distinct_count)
- statistics.__set_distinct_count(row_group_statistics.distinct_count);
- return statistics;
-}
-
-// This subclass delimits pages appearing in a serialized stream, each preceded
-// by a serialized Thrift format::PageHeader indicating the type of each page
-// and the page metadata.
-class SerializedPageWriter : public PageWriter {
- public:
- SerializedPageWriter(OutputStream* sink, Compression::type codec,
- ColumnChunkMetaDataBuilder* metadata,
- ::arrow::MemoryPool* pool = ::arrow::default_memory_pool())
- : sink_(sink),
- metadata_(metadata),
- pool_(pool),
- num_values_(0),
- dictionary_page_offset_(0),
- data_page_offset_(0),
- total_uncompressed_size_(0),
- total_compressed_size_(0) {
- compressor_ = GetCodecFromArrow(codec);
- }
-
- int64_t WriteDictionaryPage(const DictionaryPage& page) override {
- int64_t uncompressed_size = page.size();
- std::shared_ptr<Buffer> compressed_data = nullptr;
- if (has_compressor()) {
- auto buffer = std::static_pointer_cast<ResizableBuffer>(
- AllocateBuffer(pool_, uncompressed_size));
- Compress(*(page.buffer().get()), buffer.get());
- compressed_data = std::static_pointer_cast<Buffer>(buffer);
- } else {
- compressed_data = page.buffer();
- }
-
- format::DictionaryPageHeader dict_page_header;
- dict_page_header.__set_num_values(page.num_values());
- dict_page_header.__set_encoding(ToThrift(page.encoding()));
- dict_page_header.__set_is_sorted(page.is_sorted());
-
- format::PageHeader page_header;
- page_header.__set_type(format::PageType::DICTIONARY_PAGE);
- page_header.__set_uncompressed_page_size(static_cast<int32_t>(uncompressed_size));
- page_header.__set_compressed_page_size(static_cast<int32_t>(compressed_data->size()));
- page_header.__set_dictionary_page_header(dict_page_header);
- // TODO(PARQUET-594) crc checksum
-
- int64_t start_pos = sink_->Tell();
- if (dictionary_page_offset_ == 0) {
- dictionary_page_offset_ = start_pos;
- }
- int64_t header_size =
- SerializeThriftMsg(&page_header, sizeof(format::PageHeader), sink_);
- sink_->Write(compressed_data->data(), compressed_data->size());
-
- total_uncompressed_size_ += uncompressed_size + header_size;
- total_compressed_size_ += compressed_data->size() + header_size;
-
- return sink_->Tell() - start_pos;
- }
-
- void Close(bool has_dictionary, bool fallback) override {
- // index_page_offset = -1 since they are not supported
- metadata_->Finish(num_values_, dictionary_page_offset_, -1, data_page_offset_,
- total_compressed_size_, total_uncompressed_size_, has_dictionary,
- fallback);
-
- // Write metadata at end of column chunk
- metadata_->WriteTo(sink_);
- }
-
- /**
- * Compress a buffer.
- */
- void Compress(const Buffer& src_buffer, ResizableBuffer* dest_buffer) override {
- DCHECK(compressor_ != nullptr);
-
- // Compress the data
- int64_t max_compressed_size =
- compressor_->MaxCompressedLen(src_buffer.size(), src_buffer.data());
-
- // Use Arrow::Buffer::shrink_to_fit = false
- // underlying buffer only keeps growing. Resize to a smaller size does not reallocate.
- PARQUET_THROW_NOT_OK(dest_buffer->Resize(max_compressed_size, false));
-
- int64_t compressed_size;
- PARQUET_THROW_NOT_OK(
- compressor_->Compress(src_buffer.size(), src_buffer.data(), max_compressed_size,
- dest_buffer->mutable_data(), &compressed_size));
- PARQUET_THROW_NOT_OK(dest_buffer->Resize(compressed_size, false));
- }
-
- int64_t WriteDataPage(const CompressedDataPage& page) override {
- int64_t uncompressed_size = page.uncompressed_size();
- std::shared_ptr<Buffer> compressed_data = page.buffer();
-
- format::DataPageHeader data_page_header;
- data_page_header.__set_num_values(page.num_values());
- data_page_header.__set_encoding(ToThrift(page.encoding()));
- data_page_header.__set_definition_level_encoding(
- ToThrift(page.definition_level_encoding()));
- data_page_header.__set_repetition_level_encoding(
- ToThrift(page.repetition_level_encoding()));
- data_page_header.__set_statistics(ToThrift(page.statistics()));
-
- format::PageHeader page_header;
- page_header.__set_type(format::PageType::DATA_PAGE);
- page_header.__set_uncompressed_page_size(static_cast<int32_t>(uncompressed_size));
- page_header.__set_compressed_page_size(static_cast<int32_t>(compressed_data->size()));
- page_header.__set_data_page_header(data_page_header);
- // TODO(PARQUET-594) crc checksum
-
- int64_t start_pos = sink_->Tell();
- if (data_page_offset_ == 0) {
- data_page_offset_ = start_pos;
- }
-
- int64_t header_size =
- SerializeThriftMsg(&page_header, sizeof(format::PageHeader), sink_);
- sink_->Write(compressed_data->data(), compressed_data->size());
-
- total_uncompressed_size_ += uncompressed_size + header_size;
- total_compressed_size_ += compressed_data->size() + header_size;
- num_values_ += page.num_values();
-
- return sink_->Tell() - start_pos;
- }
-
- bool has_compressor() override { return (compressor_ != nullptr); }
-
- int64_t num_values() { return num_values_; }
-
- int64_t dictionary_page_offset() { return dictionary_page_offset_; }
-
- int64_t data_page_offset() { return data_page_offset_; }
-
- int64_t total_compressed_size() { return total_compressed_size_; }
-
- int64_t total_uncompressed_size() { return total_uncompressed_size_; }
-
- private:
- OutputStream* sink_;
- ColumnChunkMetaDataBuilder* metadata_;
- ::arrow::MemoryPool* pool_;
- int64_t num_values_;
- int64_t dictionary_page_offset_;
- int64_t data_page_offset_;
- int64_t total_uncompressed_size_;
- int64_t total_compressed_size_;
-
- // Compression codec to use.
- std::unique_ptr<::arrow::Codec> compressor_;
-};
-
-// This implementation of the PageWriter writes to the final sink on Close .
-class BufferedPageWriter : public PageWriter {
- public:
- BufferedPageWriter(OutputStream* sink, Compression::type codec,
- ColumnChunkMetaDataBuilder* metadata,
- ::arrow::MemoryPool* pool = ::arrow::default_memory_pool())
- : final_sink_(sink),
- metadata_(metadata),
- in_memory_sink_(new InMemoryOutputStream(pool)),
- pager_(new SerializedPageWriter(in_memory_sink_.get(), codec, metadata, pool)) {}
-
- int64_t WriteDictionaryPage(const DictionaryPage& page) override {
- return pager_->WriteDictionaryPage(page);
- }
-
- void Close(bool has_dictionary, bool fallback) override {
- // index_page_offset = -1 since they are not supported
- metadata_->Finish(
- pager_->num_values(), pager_->dictionary_page_offset() + final_sink_->Tell(), -1,
- pager_->data_page_offset() + final_sink_->Tell(), pager_->total_compressed_size(),
- pager_->total_uncompressed_size(), has_dictionary, fallback);
-
- // Write metadata at end of column chunk
- metadata_->WriteTo(in_memory_sink_.get());
-
- // flush everything to the serialized sink
- auto buffer = in_memory_sink_->GetBuffer();
- final_sink_->Write(buffer->data(), buffer->size());
- }
-
- int64_t WriteDataPage(const CompressedDataPage& page) override {
- return pager_->WriteDataPage(page);
- }
-
- void Compress(const Buffer& src_buffer, ResizableBuffer* dest_buffer) override {
- pager_->Compress(src_buffer, dest_buffer);
- }
-
- bool has_compressor() override { return pager_->has_compressor(); }
-
- private:
- OutputStream* final_sink_;
- ColumnChunkMetaDataBuilder* metadata_;
- std::unique_ptr<InMemoryOutputStream> in_memory_sink_;
- std::unique_ptr<SerializedPageWriter> pager_;
-};
-
-std::unique_ptr<PageWriter> PageWriter::Open(OutputStream* sink, Compression::type codec,
- ColumnChunkMetaDataBuilder* metadata,
- ::arrow::MemoryPool* pool,
- bool buffered_row_group) {
- if (buffered_row_group) {
- return std::unique_ptr<PageWriter>(
- new BufferedPageWriter(sink, codec, metadata, pool));
- } else {
- return std::unique_ptr<PageWriter>(
- new SerializedPageWriter(sink, codec, metadata, pool));
- }
-}
-
-// ----------------------------------------------------------------------
-// ColumnWriter
-
-std::shared_ptr<WriterProperties> default_writer_properties() {
- static std::shared_ptr<WriterProperties> default_writer_properties =
- WriterProperties::Builder().build();
- return default_writer_properties;
-}
-
-ColumnWriter::ColumnWriter(ColumnChunkMetaDataBuilder* metadata,
- std::unique_ptr<PageWriter> pager, bool has_dictionary,
- Encoding::type encoding, const WriterProperties* properties)
- : metadata_(metadata),
- descr_(metadata->descr()),
- pager_(std::move(pager)),
- has_dictionary_(has_dictionary),
- encoding_(encoding),
- properties_(properties),
- allocator_(properties->memory_pool()),
- pool_(properties->memory_pool()),
- num_buffered_values_(0),
- num_buffered_encoded_values_(0),
- rows_written_(0),
- total_bytes_written_(0),
- total_compressed_bytes_(0),
- closed_(false),
- fallback_(false) {
- definition_levels_sink_.reset(new InMemoryOutputStream(allocator_));
- repetition_levels_sink_.reset(new InMemoryOutputStream(allocator_));
- definition_levels_rle_ =
- std::static_pointer_cast<ResizableBuffer>(AllocateBuffer(allocator_, 0));
- repetition_levels_rle_ =
- std::static_pointer_cast<ResizableBuffer>(AllocateBuffer(allocator_, 0));
- uncompressed_data_ =
- std::static_pointer_cast<ResizableBuffer>(AllocateBuffer(allocator_, 0));
- if (pager_->has_compressor()) {
- compressed_data_ =
- std::static_pointer_cast<ResizableBuffer>(AllocateBuffer(allocator_, 0));
- }
-}
-
-void ColumnWriter::InitSinks() {
- definition_levels_sink_->Clear();
- repetition_levels_sink_->Clear();
-}
-
-void ColumnWriter::WriteDefinitionLevels(int64_t num_levels, const int16_t* levels) {
- DCHECK(!closed_);
- definition_levels_sink_->Write(reinterpret_cast<const uint8_t*>(levels),
- sizeof(int16_t) * num_levels);
-}
-
-void ColumnWriter::WriteRepetitionLevels(int64_t num_levels, const int16_t* levels) {
- DCHECK(!closed_);
- repetition_levels_sink_->Write(reinterpret_cast<const uint8_t*>(levels),
- sizeof(int16_t) * num_levels);
-}
-
-// return the size of the encoded buffer
-int64_t ColumnWriter::RleEncodeLevels(const Buffer& src_buffer,
- ResizableBuffer* dest_buffer, int16_t max_level) {
- // TODO: This only works with due to some RLE specifics
- int64_t rle_size = LevelEncoder::MaxBufferSize(Encoding::RLE, max_level,
- static_cast<int>(num_buffered_values_)) +
- sizeof(int32_t);
-
- // Use Arrow::Buffer::shrink_to_fit = false
- // underlying buffer only keeps growing. Resize to a smaller size does not reallocate.
- PARQUET_THROW_NOT_OK(dest_buffer->Resize(rle_size, false));
-
- level_encoder_.Init(Encoding::RLE, max_level, static_cast<int>(num_buffered_values_),
- dest_buffer->mutable_data() + sizeof(int32_t),
- static_cast<int>(dest_buffer->size() - sizeof(int32_t)));
- int encoded =
- level_encoder_.Encode(static_cast<int>(num_buffered_values_),
- reinterpret_cast<const int16_t*>(src_buffer.data()));
- DCHECK_EQ(encoded, num_buffered_values_);
- reinterpret_cast<int32_t*>(dest_buffer->mutable_data())[0] = level_encoder_.len();
- int64_t encoded_size = level_encoder_.len() + sizeof(int32_t);
- return encoded_size;
-}
-
-void ColumnWriter::AddDataPage() {
- int64_t definition_levels_rle_size = 0;
- int64_t repetition_levels_rle_size = 0;
-
- std::shared_ptr<Buffer> values = GetValuesBuffer();
-
- if (descr_->max_definition_level() > 0) {
- definition_levels_rle_size =
- RleEncodeLevels(definition_levels_sink_->GetBufferRef(),
- definition_levels_rle_.get(), descr_->max_definition_level());
- }
-
- if (descr_->max_repetition_level() > 0) {
- repetition_levels_rle_size =
- RleEncodeLevels(repetition_levels_sink_->GetBufferRef(),
- repetition_levels_rle_.get(), descr_->max_repetition_level());
- }
-
- int64_t uncompressed_size =
- definition_levels_rle_size + repetition_levels_rle_size + values->size();
-
- // Use Arrow::Buffer::shrink_to_fit = false
- // underlying buffer only keeps growing. Resize to a smaller size does not reallocate.
- PARQUET_THROW_NOT_OK(uncompressed_data_->Resize(uncompressed_size, false));
-
- // Concatenate data into a single buffer
- uint8_t* uncompressed_ptr = uncompressed_data_->mutable_data();
- memcpy(uncompressed_ptr, repetition_levels_rle_->data(), repetition_levels_rle_size);
- uncompressed_ptr += repetition_levels_rle_size;
- memcpy(uncompressed_ptr, definition_levels_rle_->data(), definition_levels_rle_size);
- uncompressed_ptr += definition_levels_rle_size;
- memcpy(uncompressed_ptr, values->data(), values->size());
-
- EncodedStatistics page_stats = GetPageStatistics();
- ResetPageStatistics();
-
- std::shared_ptr<Buffer> compressed_data;
- if (pager_->has_compressor()) {
- pager_->Compress(*(uncompressed_data_.get()), compressed_data_.get());
- compressed_data = compressed_data_;
- } else {
- compressed_data = uncompressed_data_;
- }
-
- // Write the page to OutputStream eagerly if there is no dictionary or
- // if dictionary encoding has fallen back to PLAIN
- if (has_dictionary_ && !fallback_) { // Save pages until end of dictionary encoding
- std::shared_ptr<Buffer> compressed_data_copy;
- PARQUET_THROW_NOT_OK(compressed_data->Copy(0, compressed_data->size(), allocator_,
- &compressed_data_copy));
- CompressedDataPage page(compressed_data_copy,
- static_cast<int32_t>(num_buffered_values_), encoding_,
- Encoding::RLE, Encoding::RLE, uncompressed_size, page_stats);
- total_compressed_bytes_ += page.size() + sizeof(format::PageHeader);
- data_pages_.push_back(std::move(page));
- } else { // Eagerly write pages
- CompressedDataPage page(compressed_data, static_cast<int32_t>(num_buffered_values_),
- encoding_, Encoding::RLE, Encoding::RLE, uncompressed_size,
- page_stats);
- WriteDataPage(page);
- }
-
- // Re-initialize the sinks for next Page.
- InitSinks();
- num_buffered_values_ = 0;
- num_buffered_encoded_values_ = 0;
-}
-
-void ColumnWriter::WriteDataPage(const CompressedDataPage& page) {
- total_bytes_written_ += pager_->WriteDataPage(page);
-}
-
-int64_t ColumnWriter::Close() {
- if (!closed_) {
- closed_ = true;
- if (has_dictionary_ && !fallback_) {
- WriteDictionaryPage();
- }
-
- FlushBufferedDataPages();
-
- EncodedStatistics chunk_statistics = GetChunkStatistics();
- // Write stats only if the column has at least one row written
- // From parquet-mr
- // Don't write stats larger than the max size rather than truncating. The
- // rationale is that some engines may use the minimum value in the page as
- // the true minimum for aggregations and there is no way to mark that a
- // value has been truncated and is a lower bound and not in the page.
- if (rows_written_ > 0 && chunk_statistics.is_set() &&
- chunk_statistics.max_stat_length() <=
- properties_->max_statistics_size(descr_->path())) {
- metadata_->SetStatistics(SortOrder::SIGNED == descr_->sort_order(),
- chunk_statistics);
- }
- pager_->Close(has_dictionary_, fallback_);
- }
-
- return total_bytes_written_;
-}
-
-void ColumnWriter::FlushBufferedDataPages() {
- // Write all outstanding data to a new page
- if (num_buffered_values_ > 0) {
- AddDataPage();
- }
- for (size_t i = 0; i < data_pages_.size(); i++) {
- WriteDataPage(data_pages_[i]);
- }
- data_pages_.clear();
- total_compressed_bytes_ = 0;
-}
-
-// ----------------------------------------------------------------------
-// TypedColumnWriter
-
-template <typename Type>
-TypedColumnWriter<Type>::TypedColumnWriter(ColumnChunkMetaDataBuilder* metadata,
- std::unique_ptr<PageWriter> pager,
- Encoding::type encoding,
- const WriterProperties* properties)
- : ColumnWriter(metadata, std::move(pager),
- (encoding == Encoding::PLAIN_DICTIONARY ||
- encoding == Encoding::RLE_DICTIONARY),
- encoding, properties) {
- switch (encoding) {
- case Encoding::PLAIN:
- current_encoder_.reset(new PlainEncoder<Type>(descr_, properties->memory_pool()));
- break;
- case Encoding::PLAIN_DICTIONARY:
- case Encoding::RLE_DICTIONARY:
- current_encoder_.reset(
- new DictEncoder<Type>(descr_, &pool_, properties->memory_pool()));
- break;
- default:
- ParquetException::NYI("Selected encoding is not supported");
- }
-
- if (properties->statistics_enabled(descr_->path()) &&
- (SortOrder::UNKNOWN != descr_->sort_order())) {
- page_statistics_ = std::unique_ptr<TypedStats>(new TypedStats(descr_, allocator_));
- chunk_statistics_ = std::unique_ptr<TypedStats>(new TypedStats(descr_, allocator_));
- }
-}
-
-// Only one Dictionary Page is written.
-// Fallback to PLAIN if dictionary page limit is reached.
-template <typename Type>
-void TypedColumnWriter<Type>::CheckDictionarySizeLimit() {
- auto dict_encoder = static_cast<DictEncoder<Type>*>(current_encoder_.get());
- if (dict_encoder->dict_encoded_size() >= properties_->dictionary_pagesize_limit()) {
- WriteDictionaryPage();
- // Serialize the buffered Dictionary Indicies
- FlushBufferedDataPages();
- fallback_ = true;
- // Only PLAIN encoding is supported for fallback in V1
- current_encoder_.reset(new PlainEncoder<Type>(descr_, properties_->memory_pool()));
- encoding_ = Encoding::PLAIN;
- }
-}
-
-template <typename Type>
-void TypedColumnWriter<Type>::WriteDictionaryPage() {
- auto dict_encoder = static_cast<DictEncoder<Type>*>(current_encoder_.get());
- std::shared_ptr<ResizableBuffer> buffer =
- AllocateBuffer(properties_->memory_pool(), dict_encoder->dict_encoded_size());
- dict_encoder->WriteDict(buffer->mutable_data());
- // TODO Get rid of this deep call
- dict_encoder->mem_pool()->FreeAll();
-
- DictionaryPage page(buffer, dict_encoder->num_entries(),
- properties_->dictionary_index_encoding());
- total_bytes_written_ += pager_->WriteDictionaryPage(page);
-}
-
-template <typename Type>
-EncodedStatistics TypedColumnWriter<Type>::GetPageStatistics() {
- EncodedStatistics result;
- if (page_statistics_) result = page_statistics_->Encode();
- return result;
-}
-
-template <typename Type>
-EncodedStatistics TypedColumnWriter<Type>::GetChunkStatistics() {
- EncodedStatistics result;
- if (chunk_statistics_) result = chunk_statistics_->Encode();
- return result;
-}
-
-template <typename Type>
-void TypedColumnWriter<Type>::ResetPageStatistics() {
- if (chunk_statistics_ != nullptr) {
- chunk_statistics_->Merge(*page_statistics_);
- page_statistics_->Reset();
- }
-}
-
-// ----------------------------------------------------------------------
-// Dynamic column writer constructor
-
-std::shared_ptr<ColumnWriter> ColumnWriter::Make(ColumnChunkMetaDataBuilder* metadata,
- std::unique_ptr<PageWriter> pager,
- const WriterProperties* properties) {
- const ColumnDescriptor* descr = metadata->descr();
- Encoding::type encoding = properties->encoding(descr->path());
- if (properties->dictionary_enabled(descr->path()) &&
- descr->physical_type() != Type::BOOLEAN) {
- encoding = properties->dictionary_page_encoding();
- }
- switch (descr->physical_type()) {
- case Type::BOOLEAN:
- return std::make_shared<BoolWriter>(metadata, std::move(pager), encoding,
- properties);
- case Type::INT32:
- return std::make_shared<Int32Writer>(metadata, std::move(pager), encoding,
- properties);
- case Type::INT64:
- return std::make_shared<Int64Writer>(metadata, std::move(pager), encoding,
- properties);
- case Type::INT96:
- return std::make_shared<Int96Writer>(metadata, std::move(pager), encoding,
- properties);
- case Type::FLOAT:
- return std::make_shared<FloatWriter>(metadata, std::move(pager), encoding,
- properties);
- case Type::DOUBLE:
- return std::make_shared<DoubleWriter>(metadata, std::move(pager), encoding,
- properties);
- case Type::BYTE_ARRAY:
- return std::make_shared<ByteArrayWriter>(metadata, std::move(pager), encoding,
- properties);
- case Type::FIXED_LEN_BYTE_ARRAY:
- return std::make_shared<FixedLenByteArrayWriter>(metadata, std::move(pager),
- encoding, properties);
- default:
- ParquetException::NYI("type reader not implemented");
- }
- // Unreachable code, but supress compiler warning
- return std::shared_ptr<ColumnWriter>(nullptr);
-}
-
-// ----------------------------------------------------------------------
-// Instantiate templated classes
-
-template <typename DType>
-inline int64_t TypedColumnWriter<DType>::WriteMiniBatch(int64_t num_values,
- const int16_t* def_levels,
- const int16_t* rep_levels,
- const T* values) {
- int64_t values_to_write = 0;
- // If the field is required and non-repeated, there are no definition levels
- if (descr_->max_definition_level() > 0) {
- for (int64_t i = 0; i < num_values; ++i) {
- if (def_levels[i] == descr_->max_definition_level()) {
- ++values_to_write;
- }
- }
-
- WriteDefinitionLevels(num_values, def_levels);
- } else {
- // Required field, write all values
- values_to_write = num_values;
- }
-
- // Not present for non-repeated fields
- if (descr_->max_repetition_level() > 0) {
- // A row could include more than one value
- // Count the occasions where we start a new row
- for (int64_t i = 0; i < num_values; ++i) {
- if (rep_levels[i] == 0) {
- rows_written_++;
- }
- }
-
- WriteRepetitionLevels(num_values, rep_levels);
- } else {
- // Each value is exactly one row
- rows_written_ += static_cast<int>(num_values);
- }
-
- // PARQUET-780
- if (values_to_write > 0) {
- DCHECK(nullptr != values) << "Values ptr cannot be NULL";
- }
-
- WriteValues(values_to_write, values);
-
- if (page_statistics_ != nullptr) {
- page_statistics_->Update(values, values_to_write, num_values - values_to_write);
- }
-
- num_buffered_values_ += num_values;
- num_buffered_encoded_values_ += values_to_write;
-
- if (current_encoder_->EstimatedDataEncodedSize() >= properties_->data_pagesize()) {
- AddDataPage();
- }
- if (has_dictionary_ && !fallback_) {
- CheckDictionarySizeLimit();
- }
-
- return values_to_write;
-}
-
-template <typename DType>
-inline int64_t TypedColumnWriter<DType>::WriteMiniBatchSpaced(
- int64_t num_values, const int16_t* def_levels, const int16_t* rep_levels,
- const uint8_t* valid_bits, int64_t valid_bits_offset, const T* values,
- int64_t* num_spaced_written) {
- int64_t values_to_write = 0;
- int64_t spaced_values_to_write = 0;
- // If the field is required and non-repeated, there are no definition levels
- if (descr_->max_definition_level() > 0) {
- // Minimal definition level for which spaced values are written
- int16_t min_spaced_def_level = descr_->max_definition_level();
- if (descr_->schema_node()->is_optional()) {
- min_spaced_def_level--;
- }
- for (int64_t i = 0; i < num_values; ++i) {
- if (def_levels[i] == descr_->max_definition_level()) {
- ++values_to_write;
- }
- if (def_levels[i] >= min_spaced_def_level) {
- ++spaced_values_to_write;
- }
- }
-
- WriteDefinitionLevels(num_values, def_levels);
- } else {
- // Required field, write all values
- values_to_write = num_values;
- spaced_values_to_write = num_values;
- }
-
- // Not present for non-repeated fields
- if (descr_->max_repetition_level() > 0) {
- // A row could include more than one value
- // Count the occasions where we start a new row
- for (int64_t i = 0; i < num_values; ++i) {
- if (rep_levels[i] == 0) {
- rows_written_++;
- }
- }
-
- WriteRepetitionLevels(num_values, rep_levels);
- } else {
- // Each value is exactly one row
- rows_written_ += static_cast<int>(num_values);
- }
-
- if (descr_->schema_node()->is_optional()) {
- WriteValuesSpaced(spaced_values_to_write, valid_bits, valid_bits_offset, values);
- } else {
- WriteValues(values_to_write, values);
- }
- *num_spaced_written = spaced_values_to_write;
-
- if (page_statistics_ != nullptr) {
- page_statistics_->UpdateSpaced(values, valid_bits, valid_bits_offset, values_to_write,
- num_values - values_to_write);
- }
-
- num_buffered_values_ += num_values;
- num_buffered_encoded_values_ += values_to_write;
-
- if (current_encoder_->EstimatedDataEncodedSize() >= properties_->data_pagesize()) {
- AddDataPage();
- }
- if (has_dictionary_ && !fallback_) {
- CheckDictionarySizeLimit();
- }
-
- return values_to_write;
-}
-
-template <typename DType>
-void TypedColumnWriter<DType>::WriteBatch(int64_t num_values, const int16_t* def_levels,
- const int16_t* rep_levels, const T* values) {
- // We check for DataPage limits only after we have inserted the values. If a user
- // writes a large number of values, the DataPage size can be much above the limit.
- // The purpose of this chunking is to bound this. Even if a user writes large number
- // of values, the chunking will ensure the AddDataPage() is called at a reasonable
- // pagesize limit
- int64_t write_batch_size = properties_->write_batch_size();
- int num_batches = static_cast<int>(num_values / write_batch_size);
- int64_t num_remaining = num_values % write_batch_size;
- int64_t value_offset = 0;
- for (int round = 0; round < num_batches; round++) {
- int64_t offset = round * write_batch_size;
- int64_t num_values = WriteMiniBatch(write_batch_size, &def_levels[offset],
- &rep_levels[offset], &values[value_offset]);
- value_offset += num_values;
- }
- // Write the remaining values
- int64_t offset = num_batches * write_batch_size;
- WriteMiniBatch(num_remaining, &def_levels[offset], &rep_levels[offset],
- &values[value_offset]);
-}
-
-template <typename DType>
-void TypedColumnWriter<DType>::WriteBatchSpaced(
- int64_t num_values, const int16_t* def_levels, const int16_t* rep_levels,
- const uint8_t* valid_bits, int64_t valid_bits_offset, const T* values) {
- // We check for DataPage limits only after we have inserted the values. If a user
- // writes a large number of values, the DataPage size can be much above the limit.
- // The purpose of this chunking is to bound this. Even if a user writes large number
- // of values, the chunking will ensure the AddDataPage() is called at a reasonable
- // pagesize limit
- int64_t write_batch_size = properties_->write_batch_size();
- int num_batches = static_cast<int>(num_values / write_batch_size);
- int64_t num_remaining = num_values % write_batch_size;
- int64_t num_spaced_written = 0;
- int64_t values_offset = 0;
- for (int round = 0; round < num_batches; round++) {
- int64_t offset = round * write_batch_size;
- WriteMiniBatchSpaced(write_batch_size, &def_levels[offset], &rep_levels[offset],
- valid_bits, valid_bits_offset + values_offset,
- values + values_offset, &num_spaced_written);
- values_offset += num_spaced_written;
- }
- // Write the remaining values
- int64_t offset = num_batches * write_batch_size;
- WriteMiniBatchSpaced(num_remaining, &def_levels[offset], &rep_levels[offset],
- valid_bits, valid_bits_offset + values_offset,
- values + values_offset, &num_spaced_written);
-}
-
-template <typename DType>
-void TypedColumnWriter<DType>::WriteValues(int64_t num_values, const T* values) {
- current_encoder_->Put(values, static_cast<int>(num_values));
-}
-
-template <typename DType>
-void TypedColumnWriter<DType>::WriteValuesSpaced(int64_t num_values,
- const uint8_t* valid_bits,
- int64_t valid_bits_offset,
- const T* values) {
- current_encoder_->PutSpaced(values, static_cast<int>(num_values), valid_bits,
- valid_bits_offset);
-}
-
-template class PARQUET_TEMPLATE_EXPORT TypedColumnWriter<BooleanType>;
-template class PARQUET_TEMPLATE_EXPORT TypedColumnWriter<Int32Type>;
-template class PARQUET_TEMPLATE_EXPORT TypedColumnWriter<Int64Type>;
-template class PARQUET_TEMPLATE_EXPORT TypedColumnWriter<Int96Type>;
-template class PARQUET_TEMPLATE_EXPORT TypedColumnWriter<FloatType>;
-template class PARQUET_TEMPLATE_EXPORT TypedColumnWriter<DoubleType>;
-template class PARQUET_TEMPLATE_EXPORT TypedColumnWriter<ByteArrayType>;
-template class PARQUET_TEMPLATE_EXPORT TypedColumnWriter<FLBAType>;
-
-} // namespace parquet
diff --git a/src/parquet/column_writer.h b/src/parquet/column_writer.h
deleted file mode 100644
index 1ba428a..0000000
--- a/src/parquet/column_writer.h
+++ /dev/null
@@ -1,328 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#ifndef PARQUET_COLUMN_WRITER_H
-#define PARQUET_COLUMN_WRITER_H
-
-#include <vector>
-
-#include "parquet/column_page.h"
-#include "parquet/encoding.h"
-#include "parquet/metadata.h"
-#include "parquet/properties.h"
-#include "parquet/schema.h"
-#include "parquet/statistics.h"
-#include "parquet/types.h"
-#include "parquet/util/memory.h"
-#include "parquet/util/visibility.h"
-
-namespace arrow {
-
-class BitWriter;
-class RleEncoder;
-
-} // namespace arrow
-
-namespace parquet {
-
-class PARQUET_EXPORT LevelEncoder {
- public:
- LevelEncoder();
- ~LevelEncoder();
-
- static int MaxBufferSize(Encoding::type encoding, int16_t max_level,
- int num_buffered_values);
-
- // Initialize the LevelEncoder.
- void Init(Encoding::type encoding, int16_t max_level, int num_buffered_values,
- uint8_t* data, int data_size);
-
- // Encodes a batch of levels from an array and returns the number of levels encoded
- int Encode(int batch_size, const int16_t* levels);
-
- int32_t len() {
- if (encoding_ != Encoding::RLE) {
- throw ParquetException("Only implemented for RLE encoding");
- }
- return rle_length_;
- }
-
- private:
- int bit_width_;
- int rle_length_;
- Encoding::type encoding_;
- std::unique_ptr<::arrow::RleEncoder> rle_encoder_;
- std::unique_ptr<::arrow::BitWriter> bit_packed_encoder_;
-};
-
-class PageWriter {
- public:
- virtual ~PageWriter() {}
-
- static std::unique_ptr<PageWriter> Open(
- OutputStream* sink, Compression::type codec, ColumnChunkMetaDataBuilder* metadata,
- ::arrow::MemoryPool* pool = ::arrow::default_memory_pool(),
- bool buffered_row_group = false);
-
- // The Column Writer decides if dictionary encoding is used if set and
- // if the dictionary encoding has fallen back to default encoding on reaching dictionary
- // page limit
- virtual void Close(bool has_dictionary, bool fallback) = 0;
-
- virtual int64_t WriteDataPage(const CompressedDataPage& page) = 0;
-
- virtual int64_t WriteDictionaryPage(const DictionaryPage& page) = 0;
-
- virtual bool has_compressor() = 0;
-
- virtual void Compress(const Buffer& src_buffer, ResizableBuffer* dest_buffer) = 0;
-};
-
-static constexpr int WRITE_BATCH_SIZE = 1000;
-class PARQUET_EXPORT ColumnWriter {
- public:
- ColumnWriter(ColumnChunkMetaDataBuilder*, std::unique_ptr<PageWriter>,
- bool has_dictionary, Encoding::type encoding,
- const WriterProperties* properties);
-
- virtual ~ColumnWriter() = default;
-
- static std::shared_ptr<ColumnWriter> Make(ColumnChunkMetaDataBuilder*,
- std::unique_ptr<PageWriter>,
- const WriterProperties* properties);
-
- Type::type type() const { return descr_->physical_type(); }
-
- const ColumnDescriptor* descr() const { return descr_; }
-
- /**
- * Closes the ColumnWriter, commits any buffered values to pages.
- *
- * @return Total size of the column in bytes
- */
- int64_t Close();
-
- int64_t rows_written() const { return rows_written_; }
-
- // Only considers the size of the compressed pages + page header
- // Some values might be still buffered an not written to a page yet
- int64_t total_compressed_bytes() const { return total_compressed_bytes_; }
-
- int64_t total_bytes_written() const { return total_bytes_written_; }
-
- const WriterProperties* properties() { return properties_; }
-
- protected:
- virtual std::shared_ptr<Buffer> GetValuesBuffer() = 0;
-
- // Serializes Dictionary Page if enabled
- virtual void WriteDictionaryPage() = 0;
-
- // Checks if the Dictionary Page size limit is reached
- // If the limit is reached, the Dictionary and Data Pages are serialized
- // The encoding is switched to PLAIN
-
- virtual void CheckDictionarySizeLimit() = 0;
-
- // Plain-encoded statistics of the current page
- virtual EncodedStatistics GetPageStatistics() = 0;
-
- // Plain-encoded statistics of the whole chunk
- virtual EncodedStatistics GetChunkStatistics() = 0;
-
- // Merges page statistics into chunk statistics, then resets the values
- virtual void ResetPageStatistics() = 0;
-
- // Adds Data Pages to an in memory buffer in dictionary encoding mode
- // Serializes the Data Pages in other encoding modes
- void AddDataPage();
-
- // Serializes Data Pages
- void WriteDataPage(const CompressedDataPage& page);
-
- // Write multiple definition levels
- void WriteDefinitionLevels(int64_t num_levels, const int16_t* levels);
-
- // Write multiple repetition levels
- void WriteRepetitionLevels(int64_t num_levels, const int16_t* levels);
-
- // RLE encode the src_buffer into dest_buffer and return the encoded size
- int64_t RleEncodeLevels(const Buffer& src_buffer, ResizableBuffer* dest_buffer,
- int16_t max_level);
-
- // Serialize the buffered Data Pages
- void FlushBufferedDataPages();
-
- ColumnChunkMetaDataBuilder* metadata_;
- const ColumnDescriptor* descr_;
-
- std::unique_ptr<PageWriter> pager_;
-
- bool has_dictionary_;
- Encoding::type encoding_;
- const WriterProperties* properties_;
-
- LevelEncoder level_encoder_;
-
- ::arrow::MemoryPool* allocator_;
- ChunkedAllocator pool_;
-
- // The total number of values stored in the data page. This is the maximum of
- // the number of encoded definition levels or encoded values. For
- // non-repeated, required columns, this is equal to the number of encoded
- // values. For repeated or optional values, there may be fewer data values
- // than levels, and this tells you how many encoded levels there are in that
- // case.
- int64_t num_buffered_values_;
-
- // The total number of stored values. For repeated or optional values, this
- // number may be lower than num_buffered_values_.
- int64_t num_buffered_encoded_values_;
-
- // Total number of rows written with this ColumnWriter
- int rows_written_;
-
- // Records the total number of bytes written by the serializer
- int64_t total_bytes_written_;
-
- // Records the current number of compressed bytes in a column
- int64_t total_compressed_bytes_;
-
- // Flag to check if the Writer has been closed
- bool closed_;
-
- // Flag to infer if dictionary encoding has fallen back to PLAIN
- bool fallback_;
-
- std::unique_ptr<InMemoryOutputStream> definition_levels_sink_;
- std::unique_ptr<InMemoryOutputStream> repetition_levels_sink_;
-
- std::shared_ptr<ResizableBuffer> definition_levels_rle_;
- std::shared_ptr<ResizableBuffer> repetition_levels_rle_;
-
- std::shared_ptr<ResizableBuffer> uncompressed_data_;
- std::shared_ptr<ResizableBuffer> compressed_data_;
-
- std::vector<CompressedDataPage> data_pages_;
-
- private:
- void InitSinks();
-};
-
-// API to write values to a single column. This is the main client facing API.
-template <typename DType>
-class PARQUET_EXPORT TypedColumnWriter : public ColumnWriter {
- public:
- typedef typename DType::c_type T;
-
- TypedColumnWriter(ColumnChunkMetaDataBuilder* metadata,
- std::unique_ptr<PageWriter> pager, Encoding::type encoding,
- const WriterProperties* properties);
-
- // Write a batch of repetition levels, definition levels, and values to the
- // column.
- void WriteBatch(int64_t num_values, const int16_t* def_levels,
- const int16_t* rep_levels, const T* values);
-
- /// Write a batch of repetition levels, definition levels, and values to the
- /// column.
- ///
- /// In comparision to WriteBatch the length of repetition and definition levels
- /// is the same as of the number of values read for max_definition_level == 1.
- /// In the case of max_definition_level > 1, the repetition and definition
- /// levels are larger than the values but the values include the null entries
- /// with definition_level == (max_definition_level - 1). Thus we have to differentiate
- /// in the parameters of this function if the input has the length of num_values or the
- /// _number of rows in the lowest nesting level_.
- ///
- /// In the case that the most inner node in the Parquet is required, the _number of rows
- /// in the lowest nesting level_ is equal to the number of non-null values. If the
- /// inner-most schema node is optional, the _number of rows in the lowest nesting level_
- /// also includes all values with definition_level == (max_definition_level - 1).
- ///
- /// @param num_values number of levels to write.
- /// @param def_levels The Parquet definiton levels, length is num_values
- /// @param rep_levels The Parquet repetition levels, length is num_values
- /// @param valid_bits Bitmap that indicates if the row is null on the lowest nesting
- /// level. The length is number of rows in the lowest nesting level.
- /// @param valid_bits_offset The offset in bits of the valid_bits where the
- /// first relevant bit resides.
- /// @param values The values in the lowest nested level including
- /// spacing for nulls on the lowest levels; input has the length
- /// of the number of rows on the lowest nesting level.
- void WriteBatchSpaced(int64_t num_values, const int16_t* def_levels,
- const int16_t* rep_levels, const uint8_t* valid_bits,
- int64_t valid_bits_offset, const T* values);
-
- // Estimated size of the values that are not written to a page yet
- int64_t EstimatedBufferedValueBytes() const {
- return current_encoder_->EstimatedDataEncodedSize();
- }
-
- protected:
- std::shared_ptr<Buffer> GetValuesBuffer() override {
- return current_encoder_->FlushValues();
- }
- void WriteDictionaryPage() override;
- void CheckDictionarySizeLimit() override;
- EncodedStatistics GetPageStatistics() override;
- EncodedStatistics GetChunkStatistics() override;
- void ResetPageStatistics() override;
-
- private:
- int64_t WriteMiniBatch(int64_t num_values, const int16_t* def_levels,
- const int16_t* rep_levels, const T* values);
-
- int64_t WriteMiniBatchSpaced(int64_t num_values, const int16_t* def_levels,
- const int16_t* rep_levels, const uint8_t* valid_bits,
- int64_t valid_bits_offset, const T* values,
- int64_t* num_spaced_written);
-
- typedef Encoder<DType> EncoderType;
-
- // Write values to a temporary buffer before they are encoded into pages
- void WriteValues(int64_t num_values, const T* values);
- void WriteValuesSpaced(int64_t num_values, const uint8_t* valid_bits,
- int64_t valid_bits_offset, const T* values);
- std::unique_ptr<EncoderType> current_encoder_;
-
- typedef TypedRowGroupStatistics<DType> TypedStats;
- std::unique_ptr<TypedStats> page_statistics_;
- std::unique_ptr<TypedStats> chunk_statistics_;
-};
-
-typedef TypedColumnWriter<BooleanType> BoolWriter;
-typedef TypedColumnWriter<Int32Type> Int32Writer;
-typedef TypedColumnWriter<Int64Type> Int64Writer;
-typedef TypedColumnWriter<Int96Type> Int96Writer;
-typedef TypedColumnWriter<FloatType> FloatWriter;
-typedef TypedColumnWriter<DoubleType> DoubleWriter;
-typedef TypedColumnWriter<ByteArrayType> ByteArrayWriter;
-typedef TypedColumnWriter<FLBAType> FixedLenByteArrayWriter;
-
-extern template class PARQUET_EXPORT TypedColumnWriter<BooleanType>;
-extern template class PARQUET_EXPORT TypedColumnWriter<Int32Type>;
-extern template class PARQUET_EXPORT TypedColumnWriter<Int64Type>;
-extern template class PARQUET_EXPORT TypedColumnWriter<Int96Type>;
-extern template class PARQUET_EXPORT TypedColumnWriter<FloatType>;
-extern template class PARQUET_EXPORT TypedColumnWriter<DoubleType>;
-extern template class PARQUET_EXPORT TypedColumnWriter<ByteArrayType>;
-extern template class PARQUET_EXPORT TypedColumnWriter<FLBAType>;
-
-} // namespace parquet
-
-#endif // PARQUET_COLUMN_READER_H
diff --git a/src/parquet/encoding-benchmark.cc b/src/parquet/encoding-benchmark.cc
deleted file mode 100644
index 364cdba..0000000
--- a/src/parquet/encoding-benchmark.cc
+++ /dev/null
@@ -1,166 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#include "benchmark/benchmark.h"
-
-#include "parquet/encoding-internal.h"
-#include "parquet/util/memory.h"
-
-using arrow::default_memory_pool;
-using arrow::MemoryPool;
-
-namespace parquet {
-
-using schema::PrimitiveNode;
-
-namespace benchmark {
-
-std::shared_ptr<ColumnDescriptor> Int64Schema(Repetition::type repetition) {
- auto node = PrimitiveNode::Make("int64", repetition, Type::INT64);
- return std::make_shared<ColumnDescriptor>(node, repetition != Repetition::REQUIRED,
- repetition == Repetition::REPEATED);
-}
-
-static void BM_PlainEncodingBoolean(::benchmark::State& state) {
- std::vector<bool> values(state.range(0), 64);
- PlainEncoder<BooleanType> encoder(nullptr);
-
- while (state.KeepRunning()) {
- encoder.Put(values, static_cast<int>(values.size()));
- encoder.FlushValues();
- }
- state.SetBytesProcessed(state.iterations() * state.range(0) * sizeof(bool));
-}
-
-BENCHMARK(BM_PlainEncodingBoolean)->Range(1024, 65536);
-
-static void BM_PlainDecodingBoolean(::benchmark::State& state) {
- std::vector<bool> values(state.range(0), 64);
- bool* output = new bool[state.range(0)];
- PlainEncoder<BooleanType> encoder(nullptr);
- encoder.Put(values, static_cast<int>(values.size()));
- std::shared_ptr<Buffer> buf = encoder.FlushValues();
-
- while (state.KeepRunning()) {
- PlainDecoder<BooleanType> decoder(nullptr);
- decoder.SetData(static_cast<int>(values.size()), buf->data(),
- static_cast<int>(buf->size()));
- decoder.Decode(output, static_cast<int>(values.size()));
- }
-
- state.SetBytesProcessed(state.iterations() * state.range(0) * sizeof(bool));
- delete[] output;
-}
-
-BENCHMARK(BM_PlainDecodingBoolean)->Range(1024, 65536);
-
-static void BM_PlainEncodingInt64(::benchmark::State& state) {
- std::vector<int64_t> values(state.range(0), 64);
- PlainEncoder<Int64Type> encoder(nullptr);
-
- while (state.KeepRunning()) {
- encoder.Put(values.data(), static_cast<int>(values.size()));
- encoder.FlushValues();
- }
- state.SetBytesProcessed(state.iterations() * state.range(0) * sizeof(int64_t));
-}
-
-BENCHMARK(BM_PlainEncodingInt64)->Range(1024, 65536);
-
-static void BM_PlainDecodingInt64(::benchmark::State& state) {
- std::vector<int64_t> values(state.range(0), 64);
- PlainEncoder<Int64Type> encoder(nullptr);
- encoder.Put(values.data(), static_cast<int>(values.size()));
- std::shared_ptr<Buffer> buf = encoder.FlushValues();
-
- while (state.KeepRunning()) {
- PlainDecoder<Int64Type> decoder(nullptr);
- decoder.SetData(static_cast<int>(values.size()), buf->data(),
- static_cast<int>(buf->size()));
- decoder.Decode(values.data(), static_cast<int>(values.size()));
- }
- state.SetBytesProcessed(state.iterations() * state.range(0) * sizeof(int64_t));
-}
-
-BENCHMARK(BM_PlainDecodingInt64)->Range(1024, 65536);
-
-template <typename Type>
-static void DecodeDict(std::vector<typename Type::c_type>& values,
- ::benchmark::State& state) {
- typedef typename Type::c_type T;
- int num_values = static_cast<int>(values.size());
-
- ChunkedAllocator pool;
- MemoryPool* allocator = default_memory_pool();
- std::shared_ptr<ColumnDescriptor> descr = Int64Schema(Repetition::REQUIRED);
-
- DictEncoder<Type> encoder(descr.get(), &pool, allocator);
- for (int i = 0; i < num_values; ++i) {
- encoder.Put(values[i]);
- }
-
- std::shared_ptr<ResizableBuffer> dict_buffer =
- AllocateBuffer(allocator, encoder.dict_encoded_size());
-
- std::shared_ptr<ResizableBuffer> indices =
- AllocateBuffer(allocator, encoder.EstimatedDataEncodedSize());
-
- encoder.WriteDict(dict_buffer->mutable_data());
- int actual_bytes =
- encoder.WriteIndices(indices->mutable_data(), static_cast<int>(indices->size()));
-
- PARQUET_THROW_NOT_OK(indices->Resize(actual_bytes));
-
- while (state.KeepRunning()) {
- PlainDecoder<Type> dict_decoder(descr.get());
- dict_decoder.SetData(encoder.num_entries(), dict_buffer->data(),
- static_cast<int>(dict_buffer->size()));
- DictionaryDecoder<Type> decoder(descr.get());
- decoder.SetDict(&dict_decoder);
- decoder.SetData(num_values, indices->data(), static_cast<int>(indices->size()));
- decoder.Decode(values.data(), num_values);
- }
-
- state.SetBytesProcessed(state.iterations() * state.range(0) * sizeof(T));
-}
-
-static void BM_DictDecodingInt64_repeats(::benchmark::State& state) {
- typedef Int64Type Type;
- typedef typename Type::c_type T;
-
- std::vector<T> values(state.range(0), 64);
- DecodeDict<Type>(values, state);
-}
-
-BENCHMARK(BM_DictDecodingInt64_repeats)->Range(1024, 65536);
-
-static void BM_DictDecodingInt64_literals(::benchmark::State& state) {
- typedef Int64Type Type;
- typedef typename Type::c_type T;
-
- std::vector<T> values(state.range(0));
- for (size_t i = 0; i < values.size(); ++i) {
- values[i] = i;
- }
- DecodeDict<Type>(values, state);
-}
-
-BENCHMARK(BM_DictDecodingInt64_literals)->Range(1024, 65536);
-
-} // namespace benchmark
-
-} // namespace parquet
diff --git a/src/parquet/encoding-internal.h b/src/parquet/encoding-internal.h
deleted file mode 100644
index 2dfb9ff..0000000
--- a/src/parquet/encoding-internal.h
+++ /dev/null
@@ -1,979 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#ifndef PARQUET_ENCODING_INTERNAL_H
-#define PARQUET_ENCODING_INTERNAL_H
-
-#include <algorithm>
-#include <cstdint>
-#include <limits>
-#include <memory>
-#include <vector>
-
-#include "arrow/util/bit-stream-utils.h"
-#include "arrow/util/bit-util.h"
-#include "arrow/util/cpu-info.h"
-#include "arrow/util/hash-util.h"
-#include "arrow/util/macros.h"
-#include "arrow/util/rle-encoding.h"
-
-#include "parquet/encoding.h"
-#include "parquet/exception.h"
-#include "parquet/schema.h"
-#include "parquet/types.h"
-#include "parquet/util/memory.h"
-
-namespace parquet {
-
-namespace BitUtil = ::arrow::BitUtil;
-using HashUtil = ::arrow::HashUtil;
-
-class ColumnDescriptor;
-
-// ----------------------------------------------------------------------
-// Encoding::PLAIN decoder implementation
-
-template <typename DType>
-class PlainDecoder : public Decoder<DType> {
- public:
- typedef typename DType::c_type T;
- using Decoder<DType>::num_values_;
-
- explicit PlainDecoder(const ColumnDescriptor* descr)
- : Decoder<DType>(descr, Encoding::PLAIN), data_(nullptr), len_(0) {
- if (descr_ && descr_->physical_type() == Type::FIXED_LEN_BYTE_ARRAY) {
- type_length_ = descr_->type_length();
- } else {
- type_length_ = -1;
- }
- }
-
- virtual void SetData(int num_values, const uint8_t* data, int len) {
- num_values_ = num_values;
- data_ = data;
- len_ = len;
- }
-
- virtual int Decode(T* buffer, int max_values);
-
- private:
- using Decoder<DType>::descr_;
- const uint8_t* data_;
- int len_;
- int type_length_;
-};
-
-// Decode routine templated on C++ type rather than type enum
-template <typename T>
-inline int DecodePlain(const uint8_t* data, int64_t data_size, int num_values,
- int type_length, T* out) {
- int bytes_to_decode = num_values * static_cast<int>(sizeof(T));
- if (data_size < bytes_to_decode) {
- ParquetException::EofException();
- }
- memcpy(out, data, bytes_to_decode);
- return bytes_to_decode;
-}
-
-// Template specialization for BYTE_ARRAY. The written values do not own their
-// own data.
-template <>
-inline int DecodePlain<ByteArray>(const uint8_t* data, int64_t data_size, int num_values,
- int type_length, ByteArray* out) {
- int bytes_decoded = 0;
- int increment;
- for (int i = 0; i < num_values; ++i) {
- uint32_t len = out[i].len = *reinterpret_cast<const uint32_t*>(data);
- increment = static_cast<int>(sizeof(uint32_t) + len);
- if (data_size < increment) ParquetException::EofException();
- out[i].ptr = data + sizeof(uint32_t);
- data += increment;
- data_size -= increment;
- bytes_decoded += increment;
- }
- return bytes_decoded;
-}
-
-// Template specialization for FIXED_LEN_BYTE_ARRAY. The written values do not
-// own their own data.
-template <>
-inline int DecodePlain<FixedLenByteArray>(const uint8_t* data, int64_t data_size,
- int num_values, int type_length,
- FixedLenByteArray* out) {
- int bytes_to_decode = type_length * num_values;
- if (data_size < bytes_to_decode) {
- ParquetException::EofException();
- }
- for (int i = 0; i < num_values; ++i) {
- out[i].ptr = data;
- data += type_length;
- data_size -= type_length;
- }
- return bytes_to_decode;
-}
-
-template <typename DType>
-inline int PlainDecoder<DType>::Decode(T* buffer, int max_values) {
- max_values = std::min(max_values, num_values_);
- int bytes_consumed = DecodePlain<T>(data_, len_, max_values, type_length_, buffer);
- data_ += bytes_consumed;
- len_ -= bytes_consumed;
- num_values_ -= max_values;
- return max_values;
-}
-
-template <>
-class PlainDecoder<BooleanType> : public Decoder<BooleanType> {
- public:
- explicit PlainDecoder(const ColumnDescriptor* descr)
- : Decoder<BooleanType>(descr, Encoding::PLAIN) {}
-
- virtual void SetData(int num_values, const uint8_t* data, int len) {
- num_values_ = num_values;
- bit_reader_ = ::arrow::BitReader(data, len);
- }
-
- // Two flavors of bool decoding
- int Decode(uint8_t* buffer, int max_values) {
- max_values = std::min(max_values, num_values_);
- bool val;
- ::arrow::internal::BitmapWriter bit_writer(buffer, 0, max_values);
- for (int i = 0; i < max_values; ++i) {
- if (!bit_reader_.GetValue(1, &val)) {
- ParquetException::EofException();
- }
- if (val) {
- bit_writer.Set();
- }
- bit_writer.Next();
- }
- bit_writer.Finish();
- num_values_ -= max_values;
- return max_values;
- }
-
- virtual int Decode(bool* buffer, int max_values) {
- max_values = std::min(max_values, num_values_);
- if (bit_reader_.GetBatch(1, buffer, max_values) != max_values) {
- ParquetException::EofException();
- }
- num_values_ -= max_values;
- return max_values;
- }
-
- private:
- ::arrow::BitReader bit_reader_;
-};
-
-// ----------------------------------------------------------------------
-// Encoding::PLAIN encoder implementation
-
-template <typename DType>
-class PlainEncoder : public Encoder<DType> {
- public:
- typedef typename DType::c_type T;
-
- explicit PlainEncoder(const ColumnDescriptor* descr,
- ::arrow::MemoryPool* pool = ::arrow::default_memory_pool())
- : Encoder<DType>(descr, Encoding::PLAIN, pool) {
- values_sink_.reset(new InMemoryOutputStream(pool));
- }
-
- int64_t EstimatedDataEncodedSize() override { return values_sink_->Tell(); }
-
- std::shared_ptr<Buffer> FlushValues() override;
- void Put(const T* src, int num_values) override;
-
- protected:
- std::unique_ptr<InMemoryOutputStream> values_sink_;
-};
-
-template <>
-class PlainEncoder<BooleanType> : public Encoder<BooleanType> {
- public:
- explicit PlainEncoder(const ColumnDescriptor* descr,
- ::arrow::MemoryPool* pool = ::arrow::default_memory_pool())
- : Encoder<BooleanType>(descr, Encoding::PLAIN, pool),
- bits_available_(kInMemoryDefaultCapacity * 8),
- bits_buffer_(AllocateBuffer(pool, kInMemoryDefaultCapacity)),
- values_sink_(new InMemoryOutputStream(pool)) {
- bit_writer_.reset(new ::arrow::BitWriter(bits_buffer_->mutable_data(),
- static_cast<int>(bits_buffer_->size())));
- }
-
- int64_t EstimatedDataEncodedSize() override {
- return values_sink_->Tell() + bit_writer_->bytes_written();
- }
-
- std::shared_ptr<Buffer> FlushValues() override {
- if (bits_available_ > 0) {
- bit_writer_->Flush();
- values_sink_->Write(bit_writer_->buffer(), bit_writer_->bytes_written());
- bit_writer_->Clear();
- bits_available_ = static_cast<int>(bits_buffer_->size()) * 8;
- }
-
- std::shared_ptr<Buffer> buffer = values_sink_->GetBuffer();
- values_sink_.reset(new InMemoryOutputStream(this->pool_));
- return buffer;
- }
-
-#define PLAINDECODER_BOOLEAN_PUT(input_type, function_attributes) \
- void Put(input_type src, int num_values) function_attributes { \
- int bit_offset = 0; \
- if (bits_available_ > 0) { \
- int bits_to_write = std::min(bits_available_, num_values); \
- for (int i = 0; i < bits_to_write; i++) { \
- bit_writer_->PutValue(src[i], 1); \
- } \
- bits_available_ -= bits_to_write; \
- bit_offset = bits_to_write; \
- \
- if (bits_available_ == 0) { \
- bit_writer_->Flush(); \
- values_sink_->Write(bit_writer_->buffer(), bit_writer_->bytes_written()); \
- bit_writer_->Clear(); \
- } \
- } \
- \
- int bits_remaining = num_values - bit_offset; \
- while (bit_offset < num_values) { \
- bits_available_ = static_cast<int>(bits_buffer_->size()) * 8; \
- \
- int bits_to_write = std::min(bits_available_, bits_remaining); \
- for (int i = bit_offset; i < bit_offset + bits_to_write; i++) { \
- bit_writer_->PutValue(src[i], 1); \
- } \
- bit_offset += bits_to_write; \
- bits_available_ -= bits_to_write; \
- bits_remaining -= bits_to_write; \
- \
- if (bits_available_ == 0) { \
- bit_writer_->Flush(); \
- values_sink_->Write(bit_writer_->buffer(), bit_writer_->bytes_written()); \
- bit_writer_->Clear(); \
- } \
- } \
- }
-
- PLAINDECODER_BOOLEAN_PUT(const bool*, override)
- PLAINDECODER_BOOLEAN_PUT(const std::vector<bool>&, )
-
- protected:
- int bits_available_;
- std::unique_ptr<::arrow::BitWriter> bit_writer_;
- std::shared_ptr<ResizableBuffer> bits_buffer_;
- std::unique_ptr<InMemoryOutputStream> values_sink_;
-};
-
-template <typename DType>
-inline std::shared_ptr<Buffer> PlainEncoder<DType>::FlushValues() {
- std::shared_ptr<Buffer> buffer = values_sink_->GetBuffer();
- values_sink_.reset(new InMemoryOutputStream(this->pool_));
- return buffer;
-}
-
-template <typename DType>
-inline void PlainEncoder<DType>::Put(const T* buffer, int num_values) {
- values_sink_->Write(reinterpret_cast<const uint8_t*>(buffer), num_values * sizeof(T));
-}
-
-template <>
-inline void PlainEncoder<ByteArrayType>::Put(const ByteArray* src, int num_values) {
- for (int i = 0; i < num_values; ++i) {
- // Write the result to the output stream
- values_sink_->Write(reinterpret_cast<const uint8_t*>(&src[i].len), sizeof(uint32_t));
- if (src[i].len > 0) {
- DCHECK(nullptr != src[i].ptr) << "Value ptr cannot be NULL";
- }
- values_sink_->Write(reinterpret_cast<const uint8_t*>(src[i].ptr), src[i].len);
- }
-}
-
-template <>
-inline void PlainEncoder<FLBAType>::Put(const FixedLenByteArray* src, int num_values) {
- for (int i = 0; i < num_values; ++i) {
- // Write the result to the output stream
- if (descr_->type_length() > 0) {
- DCHECK(nullptr != src[i].ptr) << "Value ptr cannot be NULL";
- }
- values_sink_->Write(reinterpret_cast<const uint8_t*>(src[i].ptr),
- descr_->type_length());
- }
-}
-
-// ----------------------------------------------------------------------
-// Dictionary encoding and decoding
-
-template <typename Type>
-class DictionaryDecoder : public Decoder<Type> {
- public:
- typedef typename Type::c_type T;
-
- // Initializes the dictionary with values from 'dictionary'. The data in
- // dictionary is not guaranteed to persist in memory after this call so the
- // dictionary decoder needs to copy the data out if necessary.
- explicit DictionaryDecoder(const ColumnDescriptor* descr,
- ::arrow::MemoryPool* pool = ::arrow::default_memory_pool())
- : Decoder<Type>(descr, Encoding::RLE_DICTIONARY),
- dictionary_(0, pool),
- byte_array_data_(AllocateBuffer(pool, 0)) {}
-
- // Perform type-specific initiatialization
- void SetDict(Decoder<Type>* dictionary);
-
- void SetData(int num_values, const uint8_t* data, int len) override {
- num_values_ = num_values;
- if (len == 0) return;
- uint8_t bit_width = *data;
- ++data;
- --len;
- idx_decoder_ = ::arrow::RleDecoder(data, len, bit_width);
- }
-
- int Decode(T* buffer, int max_values) override {
- max_values = std::min(max_values, num_values_);
- int decoded_values =
- idx_decoder_.GetBatchWithDict(dictionary_.data(), buffer, max_values);
- if (decoded_values != max_values) {
- ParquetException::EofException();
- }
- num_values_ -= max_values;
- return max_values;
- }
-
- int DecodeSpaced(T* buffer, int num_values, int null_count, const uint8_t* valid_bits,
- int64_t valid_bits_offset) override {
- int decoded_values =
- idx_decoder_.GetBatchWithDictSpaced(dictionary_.data(), buffer, num_values,
- null_count, valid_bits, valid_bits_offset);
- if (decoded_values != num_values) {
- ParquetException::EofException();
- }
- return decoded_values;
- }
-
- private:
- using Decoder<Type>::num_values_;
-
- // Only one is set.
- Vector<T> dictionary_;
-
- // Data that contains the byte array data (byte_array_dictionary_ just has the
- // pointers).
- std::shared_ptr<ResizableBuffer> byte_array_data_;
-
- ::arrow::RleDecoder idx_decoder_;
-};
-
-template <typename Type>
-inline void DictionaryDecoder<Type>::SetDict(Decoder<Type>* dictionary) {
- int num_dictionary_values = dictionary->values_left();
- dictionary_.Resize(num_dictionary_values);
- dictionary->Decode(&dictionary_[0], num_dictionary_values);
-}
-
-template <>
-inline void DictionaryDecoder<BooleanType>::SetDict(Decoder<BooleanType>* dictionary) {
- ParquetException::NYI("Dictionary encoding is not implemented for boolean values");
-}
-
-template <>
-inline void DictionaryDecoder<ByteArrayType>::SetDict(
- Decoder<ByteArrayType>* dictionary) {
- int num_dictionary_values = dictionary->values_left();
- dictionary_.Resize(num_dictionary_values);
- dictionary->Decode(&dictionary_[0], num_dictionary_values);
-
- int total_size = 0;
- for (int i = 0; i < num_dictionary_values; ++i) {
- total_size += dictionary_[i].len;
- }
- if (total_size > 0) {
- PARQUET_THROW_NOT_OK(byte_array_data_->Resize(total_size, false));
- }
-
- int offset = 0;
- uint8_t* bytes_data = byte_array_data_->mutable_data();
- for (int i = 0; i < num_dictionary_values; ++i) {
- memcpy(bytes_data + offset, dictionary_[i].ptr, dictionary_[i].len);
- dictionary_[i].ptr = bytes_data + offset;
- offset += dictionary_[i].len;
- }
-}
-
-template <>
-inline void DictionaryDecoder<FLBAType>::SetDict(Decoder<FLBAType>* dictionary) {
- int num_dictionary_values = dictionary->values_left();
- dictionary_.Resize(num_dictionary_values);
- dictionary->Decode(&dictionary_[0], num_dictionary_values);
-
- int fixed_len = descr_->type_length();
- int total_size = num_dictionary_values * fixed_len;
-
- PARQUET_THROW_NOT_OK(byte_array_data_->Resize(total_size, false));
- uint8_t* bytes_data = byte_array_data_->mutable_data();
- for (int32_t i = 0, offset = 0; i < num_dictionary_values; ++i, offset += fixed_len) {
- memcpy(bytes_data + offset, dictionary_[i].ptr, fixed_len);
- dictionary_[i].ptr = bytes_data + offset;
- }
-}
-
-// ----------------------------------------------------------------------
-// Dictionary encoder
-
-// Initially imported from Apache Impala on 2016-02-22, and has been modified
-// since for parquet-cpp
-
-// Initially 1024 elements
-static constexpr int INITIAL_HASH_TABLE_SIZE = 1 << 10;
-
-typedef int32_t hash_slot_t;
-static constexpr hash_slot_t HASH_SLOT_EMPTY = std::numeric_limits<int32_t>::max();
-
-// The maximum load factor for the hash table before resizing.
-static constexpr double MAX_HASH_LOAD = 0.7;
-
-/// See the dictionary encoding section of https://github.com/Parquet/parquet-format.
-/// The encoding supports streaming encoding. Values are encoded as they are added while
-/// the dictionary is being constructed. At any time, the buffered values can be
-/// written out with the current dictionary size. More values can then be added to
-/// the encoder, including new dictionary entries.
-template <typename DType>
-class DictEncoder : public Encoder<DType> {
- public:
- typedef typename DType::c_type T;
-
- explicit DictEncoder(const ColumnDescriptor* desc, ChunkedAllocator* pool = nullptr,
- ::arrow::MemoryPool* allocator = ::arrow::default_memory_pool())
- : Encoder<DType>(desc, Encoding::PLAIN_DICTIONARY, allocator),
- allocator_(allocator),
- pool_(pool),
- hash_table_size_(INITIAL_HASH_TABLE_SIZE),
- mod_bitmask_(hash_table_size_ - 1),
- hash_slots_(0, allocator),
- dict_encoded_size_(0),
- type_length_(desc->type_length()) {
- hash_slots_.Assign(hash_table_size_, HASH_SLOT_EMPTY);
- if (!::arrow::CpuInfo::initialized()) {
- ::arrow::CpuInfo::Init();
- }
- }
-
- ~DictEncoder() override { DCHECK(buffered_indices_.empty()); }
-
- // TODO(wesm): think about how to address the construction semantics in
- // encodings/dictionary-encoding.h
- void set_mem_pool(ChunkedAllocator* pool) { pool_ = pool; }
-
- void set_type_length(int type_length) { type_length_ = type_length; }
-
- /// Returns a conservative estimate of the number of bytes needed to encode the buffered
- /// indices. Used to size the buffer passed to WriteIndices().
- int64_t EstimatedDataEncodedSize() override {
- // Note: because of the way RleEncoder::CheckBufferFull() is called, we have to
- // reserve
- // an extra "RleEncoder::MinBufferSize" bytes. These extra bytes won't be used
- // but not reserving them would cause the encoder to fail.
- return 1 +
- ::arrow::RleEncoder::MaxBufferSize(
- bit_width(), static_cast<int>(buffered_indices_.size())) +
- ::arrow::RleEncoder::MinBufferSize(bit_width());
- }
-
- /// The minimum bit width required to encode the currently buffered indices.
- int bit_width() const {
- if (ARROW_PREDICT_FALSE(num_entries() == 0)) return 0;
- if (ARROW_PREDICT_FALSE(num_entries() == 1)) return 1;
- return BitUtil::Log2(num_entries());
- }
-
- /// Writes out any buffered indices to buffer preceded by the bit width of this data.
- /// Returns the number of bytes written.
- /// If the supplied buffer is not big enough, returns -1.
- /// buffer must be preallocated with buffer_len bytes. Use EstimatedDataEncodedSize()
- /// to size buffer.
- int WriteIndices(uint8_t* buffer, int buffer_len);
-
- int hash_table_size() { return hash_table_size_; }
- int dict_encoded_size() { return dict_encoded_size_; }
- /// Clears all the indices (but leaves the dictionary).
- void ClearIndices() { buffered_indices_.clear(); }
-
- /// Encode value. Note that this does not actually write any data, just
- /// buffers the value's index to be written later.
- void Put(const T& value);
-
- std::shared_ptr<Buffer> FlushValues() override {
- std::shared_ptr<ResizableBuffer> buffer =
- AllocateBuffer(this->allocator_, EstimatedDataEncodedSize());
- int result_size = WriteIndices(buffer->mutable_data(),
- static_cast<int>(EstimatedDataEncodedSize()));
- ClearIndices();
- PARQUET_THROW_NOT_OK(buffer->Resize(result_size, false));
- return buffer;
- }
-
- void Put(const T* values, int num_values) override {
- for (int i = 0; i < num_values; i++) {
- Put(values[i]);
- }
- }
-
- void PutSpaced(const T* src, int num_values, const uint8_t* valid_bits,
- int64_t valid_bits_offset) override {
- ::arrow::internal::BitmapReader valid_bits_reader(valid_bits, valid_bits_offset,
- num_values);
- for (int32_t i = 0; i < num_values; i++) {
- if (valid_bits_reader.IsSet()) {
- Put(src[i]);
- }
- valid_bits_reader.Next();
- }
- }
-
- /// Writes out the encoded dictionary to buffer. buffer must be preallocated to
- /// dict_encoded_size() bytes.
- void WriteDict(uint8_t* buffer);
-
- ChunkedAllocator* mem_pool() { return pool_; }
-
- /// The number of entries in the dictionary.
- int num_entries() const { return static_cast<int>(uniques_.size()); }
-
- private:
- ::arrow::MemoryPool* allocator_;
-
- // For ByteArray / FixedLenByteArray data. Not owned
- ChunkedAllocator* pool_;
-
- /// Size of the table. Must be a power of 2.
- int hash_table_size_;
-
- // Store hash_table_size_ - 1, so that j & mod_bitmask_ is equivalent to j %
- // hash_table_size_, but uses far fewer CPU cycles
- int mod_bitmask_;
-
- // We use a fixed-size hash table with linear probing
- //
- // These values correspond to the uniques_ array
- Vector<hash_slot_t> hash_slots_;
-
- /// Indices that have not yet be written out by WriteIndices().
- std::vector<int> buffered_indices_;
-
- /// The number of bytes needed to encode the dictionary.
- int dict_encoded_size_;
-
- // The unique observed values
- std::vector<T> uniques_;
-
- bool SlotDifferent(const T& v, hash_slot_t slot);
- void DoubleTableSize();
-
- /// Size of each encoded dictionary value. -1 for variable-length types.
- int type_length_;
-
- /// Hash function for mapping a value to a bucket.
- inline int Hash(const T& value) const;
-
- /// Adds value to the hash table and updates dict_encoded_size_
- void AddDictKey(const T& value);
-};
-
-template <typename DType>
-inline int DictEncoder<DType>::Hash(const typename DType::c_type& value) const {
- return HashUtil::Hash(&value, sizeof(value), 0);
-}
-
-template <>
-inline int DictEncoder<ByteArrayType>::Hash(const ByteArray& value) const {
- if (value.len > 0) {
- DCHECK_NE(nullptr, value.ptr) << "Value ptr cannot be NULL";
- }
- return HashUtil::Hash(value.ptr, value.len, 0);
-}
-
-template <>
-inline int DictEncoder<FLBAType>::Hash(const FixedLenByteArray& value) const {
- if (type_length_ > 0) {
- DCHECK_NE(nullptr, value.ptr) << "Value ptr cannot be NULL";
- }
- return HashUtil::Hash(value.ptr, type_length_, 0);
-}
-
-template <typename DType>
-inline bool DictEncoder<DType>::SlotDifferent(const typename DType::c_type& v,
- hash_slot_t slot) {
- return v != uniques_[slot];
-}
-
-template <>
-inline bool DictEncoder<FLBAType>::SlotDifferent(const FixedLenByteArray& v,
- hash_slot_t slot) {
- return 0 != memcmp(v.ptr, uniques_[slot].ptr, type_length_);
-}
-
-template <typename DType>
-inline void DictEncoder<DType>::Put(const typename DType::c_type& v) {
- int j = Hash(v) & mod_bitmask_;
- hash_slot_t index = hash_slots_[j];
-
- // Find an empty slot
- while (HASH_SLOT_EMPTY != index && SlotDifferent(v, index)) {
- // Linear probing
- ++j;
- if (j == hash_table_size_) j = 0;
- index = hash_slots_[j];
- }
-
- if (index == HASH_SLOT_EMPTY) {
- // Not in the hash table, so we insert it now
- index = static_cast<hash_slot_t>(uniques_.size());
- hash_slots_[j] = index;
- AddDictKey(v);
-
- if (ARROW_PREDICT_FALSE(static_cast<int>(uniques_.size()) >
- hash_table_size_ * MAX_HASH_LOAD)) {
- DoubleTableSize();
- }
- }
-
- buffered_indices_.push_back(index);
-}
-
-template <typename DType>
-inline void DictEncoder<DType>::DoubleTableSize() {
- int new_size = hash_table_size_ * 2;
- Vector<hash_slot_t> new_hash_slots(0, allocator_);
- new_hash_slots.Assign(new_size, HASH_SLOT_EMPTY);
- hash_slot_t index, slot;
- int j;
- for (int i = 0; i < hash_table_size_; ++i) {
- index = hash_slots_[i];
-
- if (index == HASH_SLOT_EMPTY) {
- continue;
- }
-
- // Compute the hash value mod the new table size to start looking for an
- // empty slot
- const typename DType::c_type& v = uniques_[index];
-
- // Find an empty slot in the new hash table
- j = Hash(v) & (new_size - 1);
- slot = new_hash_slots[j];
- while (HASH_SLOT_EMPTY != slot && SlotDifferent(v, slot)) {
- ++j;
- if (j == new_size) j = 0;
- slot = new_hash_slots[j];
- }
-
- // Copy the old slot index to the new hash table
- new_hash_slots[j] = index;
- }
-
- hash_table_size_ = new_size;
- mod_bitmask_ = new_size - 1;
-
- hash_slots_.Swap(new_hash_slots);
-}
-
-template <typename DType>
-inline void DictEncoder<DType>::AddDictKey(const typename DType::c_type& v) {
- uniques_.push_back(v);
- dict_encoded_size_ += static_cast<int>(sizeof(typename DType::c_type));
-}
-
-template <>
-inline void DictEncoder<ByteArrayType>::AddDictKey(const ByteArray& v) {
- uint8_t* heap = pool_->Allocate(v.len);
- if (ARROW_PREDICT_FALSE(v.len > 0 && heap == nullptr)) {
- throw ParquetException("out of memory");
- }
- memcpy(heap, v.ptr, v.len);
- uniques_.push_back(ByteArray(v.len, heap));
- dict_encoded_size_ += static_cast<int>(v.len + sizeof(uint32_t));
-}
-
-template <>
-inline void DictEncoder<FLBAType>::AddDictKey(const FixedLenByteArray& v) {
- uint8_t* heap = pool_->Allocate(type_length_);
- if (ARROW_PREDICT_FALSE(type_length_ > 0 && heap == nullptr)) {
- throw ParquetException("out of memory");
- }
- memcpy(heap, v.ptr, type_length_);
-
- uniques_.push_back(FixedLenByteArray(heap));
- dict_encoded_size_ += type_length_;
-}
-
-template <typename DType>
-inline void DictEncoder<DType>::WriteDict(uint8_t* buffer) {
- // For primitive types, only a memcpy
- memcpy(buffer, uniques_.data(), sizeof(typename DType::c_type) * uniques_.size());
-}
-
-template <>
-inline void DictEncoder<BooleanType>::WriteDict(uint8_t* buffer) {
- // For primitive types, only a memcpy
- // memcpy(buffer, uniques_.data(), sizeof(typename DType::c_type) * uniques_.size());
- for (size_t i = 0; i < uniques_.size(); i++) {
- buffer[i] = uniques_[i];
- }
-}
-
-// ByteArray and FLBA already have the dictionary encoded in their data heaps
-template <>
-inline void DictEncoder<ByteArrayType>::WriteDict(uint8_t* buffer) {
- for (const ByteArray& v : uniques_) {
- memcpy(buffer, reinterpret_cast<const void*>(&v.len), sizeof(uint32_t));
- buffer += sizeof(uint32_t);
- if (v.len > 0) {
- DCHECK(nullptr != v.ptr) << "Value ptr cannot be NULL";
- }
- memcpy(buffer, v.ptr, v.len);
- buffer += v.len;
- }
-}
-
-template <>
-inline void DictEncoder<FLBAType>::WriteDict(uint8_t* buffer) {
- for (const FixedLenByteArray& v : uniques_) {
- if (type_length_ > 0) {
- DCHECK(nullptr != v.ptr) << "Value ptr cannot be NULL";
- }
- memcpy(buffer, v.ptr, type_length_);
- buffer += type_length_;
- }
-}
-
-template <typename DType>
-inline int DictEncoder<DType>::WriteIndices(uint8_t* buffer, int buffer_len) {
- // Write bit width in first byte
- *buffer = static_cast<uint8_t>(bit_width());
- ++buffer;
- --buffer_len;
-
- ::arrow::RleEncoder encoder(buffer, buffer_len, bit_width());
- for (int index : buffered_indices_) {
- if (!encoder.Put(index)) return -1;
- }
- encoder.Flush();
-
- ClearIndices();
- return 1 + encoder.len();
-}
-
-// ----------------------------------------------------------------------
-// DeltaBitPackDecoder
-
-template <typename DType>
-class DeltaBitPackDecoder : public Decoder<DType> {
- public:
- typedef typename DType::c_type T;
-
- explicit DeltaBitPackDecoder(const ColumnDescriptor* descr,
- ::arrow::MemoryPool* pool = ::arrow::default_memory_pool())
- : Decoder<DType>(descr, Encoding::DELTA_BINARY_PACKED), pool_(pool) {
- if (DType::type_num != Type::INT32 && DType::type_num != Type::INT64) {
- throw ParquetException("Delta bit pack encoding should only be for integer data.");
- }
- }
-
- virtual void SetData(int num_values, const uint8_t* data, int len) {
- num_values_ = num_values;
- decoder_ = ::arrow::BitReader(data, len);
- values_current_block_ = 0;
- values_current_mini_block_ = 0;
- }
-
- virtual int Decode(T* buffer, int max_values) {
- return GetInternal(buffer, max_values);
- }
-
- private:
- using Decoder<DType>::num_values_;
-
- void InitBlock() {
- int32_t block_size;
- if (!decoder_.GetVlqInt(&block_size)) ParquetException::EofException();
- if (!decoder_.GetVlqInt(&num_mini_blocks_)) ParquetException::EofException();
- if (!decoder_.GetVlqInt(&values_current_block_)) {
- ParquetException::EofException();
- }
- if (!decoder_.GetZigZagVlqInt(&last_value_)) ParquetException::EofException();
-
- delta_bit_widths_ = AllocateBuffer(pool_, num_mini_blocks_);
- uint8_t* bit_width_data = delta_bit_widths_->mutable_data();
-
- if (!decoder_.GetZigZagVlqInt(&min_delta_)) ParquetException::EofException();
- for (int i = 0; i < num_mini_blocks_; ++i) {
- if (!decoder_.GetAligned<uint8_t>(1, bit_width_data + i)) {
- ParquetException::EofException();
- }
- }
- values_per_mini_block_ = block_size / num_mini_blocks_;
- mini_block_idx_ = 0;
- delta_bit_width_ = bit_width_data[0];
- values_current_mini_block_ = values_per_mini_block_;
- }
-
- template <typename T>
- int GetInternal(T* buffer, int max_values) {
- max_values = std::min(max_values, num_values_);
- const uint8_t* bit_width_data = delta_bit_widths_->data();
- for (int i = 0; i < max_values; ++i) {
- if (ARROW_PREDICT_FALSE(values_current_mini_block_ == 0)) {
- ++mini_block_idx_;
- if (mini_block_idx_ < static_cast<size_t>(delta_bit_widths_->size())) {
- delta_bit_width_ = bit_width_data[mini_block_idx_];
- values_current_mini_block_ = values_per_mini_block_;
- } else {
- InitBlock();
- buffer[i] = last_value_;
- continue;
- }
- }
-
- // TODO: the key to this algorithm is to decode the entire miniblock at once.
- int64_t delta;
- if (!decoder_.GetValue(delta_bit_width_, &delta)) ParquetException::EofException();
- delta += min_delta_;
- last_value_ += static_cast<int32_t>(delta);
- buffer[i] = last_value_;
- --values_current_mini_block_;
- }
- num_values_ -= max_values;
- return max_values;
- }
-
- ::arrow::MemoryPool* pool_;
- ::arrow::BitReader decoder_;
- int32_t values_current_block_;
- int32_t num_mini_blocks_;
- uint64_t values_per_mini_block_;
- uint64_t values_current_mini_block_;
-
- int32_t min_delta_;
- size_t mini_block_idx_;
- std::shared_ptr<ResizableBuffer> delta_bit_widths_;
- int delta_bit_width_;
-
- int32_t last_value_;
-};
-
-// ----------------------------------------------------------------------
-// DELTA_LENGTH_BYTE_ARRAY
-
-class DeltaLengthByteArrayDecoder : public Decoder<ByteArrayType> {
- public:
- explicit DeltaLengthByteArrayDecoder(
- const ColumnDescriptor* descr,
- ::arrow::MemoryPool* pool = ::arrow::default_memory_pool())
- : Decoder<ByteArrayType>(descr, Encoding::DELTA_LENGTH_BYTE_ARRAY),
- len_decoder_(nullptr, pool) {}
-
- virtual void SetData(int num_values, const uint8_t* data, int len) {
- num_values_ = num_values;
- if (len == 0) return;
- int total_lengths_len = *reinterpret_cast<const int*>(data);
- data += 4;
- len_decoder_.SetData(num_values, data, total_lengths_len);
- data_ = data + total_lengths_len;
- len_ = len - 4 - total_lengths_len;
- }
-
- virtual int Decode(ByteArray* buffer, int max_values) {
- max_values = std::min(max_values, num_values_);
- std::vector<int> lengths(max_values);
- len_decoder_.Decode(lengths.data(), max_values);
- for (int i = 0; i < max_values; ++i) {
- buffer[i].len = lengths[i];
- buffer[i].ptr = data_;
- data_ += lengths[i];
- len_ -= lengths[i];
- }
- num_values_ -= max_values;
- return max_values;
- }
-
- private:
- using Decoder<ByteArrayType>::num_values_;
- DeltaBitPackDecoder<Int32Type> len_decoder_;
- const uint8_t* data_;
- int len_;
-};
-
-// ----------------------------------------------------------------------
-// DELTA_BYTE_ARRAY
-
-class DeltaByteArrayDecoder : public Decoder<ByteArrayType> {
- public:
- explicit DeltaByteArrayDecoder(
- const ColumnDescriptor* descr,
- ::arrow::MemoryPool* pool = ::arrow::default_memory_pool())
- : Decoder<ByteArrayType>(descr, Encoding::DELTA_BYTE_ARRAY),
- prefix_len_decoder_(nullptr, pool),
- suffix_decoder_(nullptr, pool),
- last_value_(0, nullptr) {}
-
- virtual void SetData(int num_values, const uint8_t* data, int len) {
- num_values_ = num_values;
- if (len == 0) return;
- int prefix_len_length = *reinterpret_cast<const int*>(data);
- data += 4;
- len -= 4;
- prefix_len_decoder_.SetData(num_values, data, prefix_len_length);
- data += prefix_len_length;
- len -= prefix_len_length;
- suffix_decoder_.SetData(num_values, data, len);
- }
-
- // TODO: this doesn't work and requires memory management. We need to allocate
- // new strings to store the results.
- virtual int Decode(ByteArray* buffer, int max_values) {
- max_values = std::min(max_values, num_values_);
- for (int i = 0; i < max_values; ++i) {
- int prefix_len = 0;
- prefix_len_decoder_.Decode(&prefix_len, 1);
- ByteArray suffix = {0, nullptr};
- suffix_decoder_.Decode(&suffix, 1);
- buffer[i].len = prefix_len + suffix.len;
-
- uint8_t* result = reinterpret_cast<uint8_t*>(malloc(buffer[i].len));
- memcpy(result, last_value_.ptr, prefix_len);
- memcpy(result + prefix_len, suffix.ptr, suffix.len);
-
- buffer[i].ptr = result;
- last_value_ = buffer[i];
- }
- num_values_ -= max_values;
- return max_values;
- }
-
- private:
- using Decoder<ByteArrayType>::num_values_;
-
- DeltaBitPackDecoder<Int32Type> prefix_len_decoder_;
- DeltaLengthByteArrayDecoder suffix_decoder_;
- ByteArray last_value_;
-};
-
-} // namespace parquet
-
-#endif // PARQUET_ENCODING_INTERNAL_H
diff --git a/src/parquet/encoding-test.cc b/src/parquet/encoding-test.cc
deleted file mode 100644
index 50e1394..0000000
--- a/src/parquet/encoding-test.cc
+++ /dev/null
@@ -1,313 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#include <gtest/gtest.h>
-#include <cstdint>
-#include <cstdlib>
-#include <cstring>
-#include <string>
-#include <vector>
-
-#include "arrow/util/bit-util.h"
-
-#include "parquet/encoding-internal.h"
-#include "parquet/schema.h"
-#include "parquet/types.h"
-#include "parquet/util/memory.h"
-#include "parquet/util/test-common.h"
-
-using arrow::default_memory_pool;
-using arrow::MemoryPool;
-
-using std::string;
-using std::vector;
-
-namespace parquet {
-
-namespace test {
-
-TEST(VectorBooleanTest, TestEncodeDecode) {
- // PARQUET-454
- int nvalues = 10000;
- int nbytes = static_cast<int>(BitUtil::BytesForBits(nvalues));
-
- // seed the prng so failure is deterministic
- vector<bool> draws = flip_coins_seed(nvalues, 0.5, 0);
-
- PlainEncoder<BooleanType> encoder(nullptr);
- PlainDecoder<BooleanType> decoder(nullptr);
-
- encoder.Put(draws, nvalues);
-
- std::shared_ptr<Buffer> encode_buffer = encoder.FlushValues();
- ASSERT_EQ(nbytes, encode_buffer->size());
-
- vector<uint8_t> decode_buffer(nbytes);
- const uint8_t* decode_data = &decode_buffer[0];
-
- decoder.SetData(nvalues, encode_buffer->data(),
- static_cast<int>(encode_buffer->size()));
- int values_decoded = decoder.Decode(&decode_buffer[0], nvalues);
- ASSERT_EQ(nvalues, values_decoded);
-
- for (int i = 0; i < nvalues; ++i) {
- ASSERT_EQ(draws[i], BitUtil::GetBit(decode_data, i)) << i;
- }
-}
-
-// ----------------------------------------------------------------------
-// test data generation
-
-template <typename T>
-void GenerateData(int num_values, T* out, vector<uint8_t>* heap) {
- // seed the prng so failure is deterministic
- random_numbers(num_values, 0, std::numeric_limits<T>::min(),
- std::numeric_limits<T>::max(), out);
-}
-
-template <>
-void GenerateData<bool>(int num_values, bool* out, vector<uint8_t>* heap) {
- // seed the prng so failure is deterministic
- random_bools(num_values, 0.5, 0, out);
-}
-
-template <>
-void GenerateData<Int96>(int num_values, Int96* out, vector<uint8_t>* heap) {
- // seed the prng so failure is deterministic
- random_Int96_numbers(num_values, 0, std::numeric_limits<int32_t>::min(),
- std::numeric_limits<int32_t>::max(), out);
-}
-
-template <>
-void GenerateData<ByteArray>(int num_values, ByteArray* out, vector<uint8_t>* heap) {
- // seed the prng so failure is deterministic
- int max_byte_array_len = 12;
- heap->resize(num_values * max_byte_array_len);
- random_byte_array(num_values, 0, heap->data(), out, 2, max_byte_array_len);
-}
-
-static int flba_length = 8;
-
-template <>
-void GenerateData<FLBA>(int num_values, FLBA* out, vector<uint8_t>* heap) {
- // seed the prng so failure is deterministic
- heap->resize(num_values * flba_length);
- random_fixed_byte_array(num_values, 0, heap->data(), flba_length, out);
-}
-
-template <typename T>
-void VerifyResults(T* result, T* expected, int num_values) {
- for (int i = 0; i < num_values; ++i) {
- ASSERT_EQ(expected[i], result[i]) << i;
- }
-}
-
-template <>
-void VerifyResults<FLBA>(FLBA* result, FLBA* expected, int num_values) {
- for (int i = 0; i < num_values; ++i) {
- ASSERT_EQ(0, memcmp(expected[i].ptr, result[i].ptr, flba_length)) << i;
- }
-}
-
-// ----------------------------------------------------------------------
-// Create some column descriptors
-
-template <typename DType>
-std::shared_ptr<ColumnDescriptor> ExampleDescr() {
- auto node = schema::PrimitiveNode::Make("name", Repetition::OPTIONAL, DType::type_num);
- return std::make_shared<ColumnDescriptor>(node, 0, 0);
-}
-
-template <>
-std::shared_ptr<ColumnDescriptor> ExampleDescr<FLBAType>() {
- auto node = schema::PrimitiveNode::Make("name", Repetition::OPTIONAL,
- Type::FIXED_LEN_BYTE_ARRAY,
- LogicalType::DECIMAL, flba_length, 10, 2);
- return std::make_shared<ColumnDescriptor>(node, 0, 0);
-}
-
-// ----------------------------------------------------------------------
-// Plain encoding tests
-
-template <typename Type>
-class TestEncodingBase : public ::testing::Test {
- public:
- typedef typename Type::c_type T;
- static constexpr int TYPE = Type::type_num;
-
- void SetUp() {
- descr_ = ExampleDescr<Type>();
- type_length_ = descr_->type_length();
- allocator_ = default_memory_pool();
- }
-
- void TearDown() { pool_.FreeAll(); }
-
- void InitData(int nvalues, int repeats) {
- num_values_ = nvalues * repeats;
- input_bytes_.resize(num_values_ * sizeof(T));
- output_bytes_.resize(num_values_ * sizeof(T));
- draws_ = reinterpret_cast<T*>(input_bytes_.data());
- decode_buf_ = reinterpret_cast<T*>(output_bytes_.data());
- GenerateData<T>(nvalues, draws_, &data_buffer_);
-
- // add some repeated values
- for (int j = 1; j < repeats; ++j) {
- for (int i = 0; i < nvalues; ++i) {
- draws_[nvalues * j + i] = draws_[i];
- }
- }
- }
-
- virtual void CheckRoundtrip() = 0;
-
- void Execute(int nvalues, int repeats) {
- InitData(nvalues, repeats);
- CheckRoundtrip();
- }
-
- protected:
- ChunkedAllocator pool_;
- MemoryPool* allocator_;
-
- int num_values_;
- int type_length_;
- T* draws_;
- T* decode_buf_;
- vector<uint8_t> input_bytes_;
- vector<uint8_t> output_bytes_;
- vector<uint8_t> data_buffer_;
-
- std::shared_ptr<Buffer> encode_buffer_;
- std::shared_ptr<ColumnDescriptor> descr_;
-};
-
-// Member variables are not visible to templated subclasses. Possibly figure
-// out an alternative to this class layering at some point
-#define USING_BASE_MEMBERS() \
- using TestEncodingBase<Type>::pool_; \
- using TestEncodingBase<Type>::allocator_; \
- using TestEncodingBase<Type>::descr_; \
- using TestEncodingBase<Type>::num_values_; \
- using TestEncodingBase<Type>::draws_; \
- using TestEncodingBase<Type>::data_buffer_; \
- using TestEncodingBase<Type>::type_length_; \
- using TestEncodingBase<Type>::encode_buffer_; \
- using TestEncodingBase<Type>::decode_buf_
-
-template <typename Type>
-class TestPlainEncoding : public TestEncodingBase<Type> {
- public:
- typedef typename Type::c_type T;
- static constexpr int TYPE = Type::type_num;
-
- virtual void CheckRoundtrip() {
- PlainEncoder<Type> encoder(descr_.get());
- PlainDecoder<Type> decoder(descr_.get());
- encoder.Put(draws_, num_values_);
- encode_buffer_ = encoder.FlushValues();
-
- decoder.SetData(num_values_, encode_buffer_->data(),
- static_cast<int>(encode_buffer_->size()));
- int values_decoded = decoder.Decode(decode_buf_, num_values_);
- ASSERT_EQ(num_values_, values_decoded);
- ASSERT_NO_FATAL_FAILURE(VerifyResults<T>(decode_buf_, draws_, num_values_));
- }
-
- protected:
- USING_BASE_MEMBERS();
-};
-
-TYPED_TEST_CASE(TestPlainEncoding, ParquetTypes);
-
-TYPED_TEST(TestPlainEncoding, BasicRoundTrip) {
- ASSERT_NO_FATAL_FAILURE(this->Execute(10000, 1));
-}
-
-// ----------------------------------------------------------------------
-// Dictionary encoding tests
-
-typedef ::testing::Types<Int32Type, Int64Type, Int96Type, FloatType, DoubleType,
- ByteArrayType, FLBAType>
- DictEncodedTypes;
-
-template <typename Type>
-class TestDictionaryEncoding : public TestEncodingBase<Type> {
- public:
- typedef typename Type::c_type T;
- static constexpr int TYPE = Type::type_num;
-
- void CheckRoundtrip() {
- std::vector<uint8_t> valid_bits(BitUtil::BytesForBits(num_values_) + 1, 255);
- DictEncoder<Type> encoder(descr_.get(), &pool_);
-
- ASSERT_NO_THROW(encoder.Put(draws_, num_values_));
- dict_buffer_ = AllocateBuffer(default_memory_pool(), encoder.dict_encoded_size());
- encoder.WriteDict(dict_buffer_->mutable_data());
- std::shared_ptr<Buffer> indices = encoder.FlushValues();
-
- DictEncoder<Type> spaced_encoder(descr_.get(), &pool_);
- // PutSpaced should lead to the same results
- ASSERT_NO_THROW(spaced_encoder.PutSpaced(draws_, num_values_, valid_bits.data(), 0));
- std::shared_ptr<Buffer> indices_from_spaced = spaced_encoder.FlushValues();
- ASSERT_TRUE(indices_from_spaced->Equals(*indices));
-
- PlainDecoder<Type> dict_decoder(descr_.get());
- dict_decoder.SetData(encoder.num_entries(), dict_buffer_->data(),
- static_cast<int>(dict_buffer_->size()));
-
- DictionaryDecoder<Type> decoder(descr_.get());
- decoder.SetDict(&dict_decoder);
-
- decoder.SetData(num_values_, indices->data(), static_cast<int>(indices->size()));
- int values_decoded = decoder.Decode(decode_buf_, num_values_);
- ASSERT_EQ(num_values_, values_decoded);
-
- // TODO(wesm): The DictionaryDecoder must stay alive because the decoded
- // values' data is owned by a buffer inside the DictionaryEncoder. We
- // should revisit when data lifetime is reviewed more generally.
- ASSERT_NO_FATAL_FAILURE(VerifyResults<T>(decode_buf_, draws_, num_values_));
-
- // Also test spaced decoding
- decoder.SetData(num_values_, indices->data(), static_cast<int>(indices->size()));
- values_decoded =
- decoder.DecodeSpaced(decode_buf_, num_values_, 0, valid_bits.data(), 0);
- ASSERT_EQ(num_values_, values_decoded);
- ASSERT_NO_FATAL_FAILURE(VerifyResults<T>(decode_buf_, draws_, num_values_));
- }
-
- protected:
- USING_BASE_MEMBERS();
- std::shared_ptr<ResizableBuffer> dict_buffer_;
-};
-
-TYPED_TEST_CASE(TestDictionaryEncoding, DictEncodedTypes);
-
-TYPED_TEST(TestDictionaryEncoding, BasicRoundTrip) {
- ASSERT_NO_FATAL_FAILURE(this->Execute(2500, 2));
-}
-
-TEST(TestDictionaryEncoding, CannotDictDecodeBoolean) {
- PlainDecoder<BooleanType> dict_decoder(nullptr);
- DictionaryDecoder<BooleanType> decoder(nullptr);
-
- ASSERT_THROW(decoder.SetDict(&dict_decoder), ParquetException);
-}
-
-} // namespace test
-
-} // namespace parquet
diff --git a/src/parquet/encoding.h b/src/parquet/encoding.h
deleted file mode 100644
index 006f22f..0000000
--- a/src/parquet/encoding.h
+++ /dev/null
@@ -1,152 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#ifndef PARQUET_ENCODING_H
-#define PARQUET_ENCODING_H
-
-#include <cstdint>
-#include <memory>
-#include <sstream>
-
-#include "arrow/status.h"
-#include "arrow/util/bit-util.h"
-
-#include "parquet/exception.h"
-#include "parquet/schema.h"
-#include "parquet/types.h"
-#include "parquet/util/memory.h"
-
-namespace parquet {
-
-class ColumnDescriptor;
-
-// Base class for value encoders. Since encoders may or not have state (e.g.,
-// dictionary encoding) we use a class instance to maintain any state.
-//
-// TODO(wesm): Encode interface API is temporary
-template <typename DType>
-class Encoder {
- public:
- typedef typename DType::c_type T;
-
- virtual ~Encoder() {}
-
- virtual int64_t EstimatedDataEncodedSize() = 0;
- virtual std::shared_ptr<Buffer> FlushValues() = 0;
- virtual void Put(const T* src, int num_values) = 0;
- virtual void PutSpaced(const T* src, int num_values, const uint8_t* valid_bits,
- int64_t valid_bits_offset) {
- std::shared_ptr<ResizableBuffer> buffer;
- auto status =
- ::arrow::AllocateResizableBuffer(pool_, num_values * sizeof(T), &buffer);
- if (!status.ok()) {
- std::ostringstream ss;
- ss << "AllocateResizableBuffer failed in Encoder.PutSpaced in " << __FILE__
- << ", on line " << __LINE__;
- throw ParquetException(ss.str());
- }
- int32_t num_valid_values = 0;
- ::arrow::internal::BitmapReader valid_bits_reader(valid_bits, valid_bits_offset,
- num_values);
- T* data = reinterpret_cast<T*>(buffer->mutable_data());
- for (int32_t i = 0; i < num_values; i++) {
- if (valid_bits_reader.IsSet()) {
- data[num_valid_values++] = src[i];
- }
- valid_bits_reader.Next();
- }
- Put(data, num_valid_values);
- }
-
- Encoding::type encoding() const { return encoding_; }
-
- protected:
- explicit Encoder(const ColumnDescriptor* descr, Encoding::type encoding,
- ::arrow::MemoryPool* pool)
- : descr_(descr), encoding_(encoding), pool_(pool) {}
-
- // For accessing type-specific metadata, like FIXED_LEN_BYTE_ARRAY
- const ColumnDescriptor* descr_;
- const Encoding::type encoding_;
- ::arrow::MemoryPool* pool_;
-};
-
-// The Decoder template is parameterized on parquet::DataType subclasses
-template <typename DType>
-class Decoder {
- public:
- typedef typename DType::c_type T;
-
- virtual ~Decoder() {}
-
- // Sets the data for a new page. This will be called multiple times on the same
- // decoder and should reset all internal state.
- virtual void SetData(int num_values, const uint8_t* data, int len) = 0;
-
- // Subclasses should override the ones they support. In each of these functions,
- // the decoder would decode put to 'max_values', storing the result in 'buffer'.
- // The function returns the number of values decoded, which should be max_values
- // except for end of the current data page.
- virtual int Decode(T* buffer, int max_values) = 0;
-
- // Decode the values in this data page but leave spaces for null entries.
- //
- // num_values is the size of the def_levels and buffer arrays including the number of
- // null values.
- virtual int DecodeSpaced(T* buffer, int num_values, int null_count,
- const uint8_t* valid_bits, int64_t valid_bits_offset) {
- int values_to_read = num_values - null_count;
- int values_read = Decode(buffer, values_to_read);
- if (values_read != values_to_read) {
- throw ParquetException("Number of values / definition_levels read did not match");
- }
-
- // Depending on the number of nulls, some of the value slots in buffer may
- // be uninitialized, and this will cause valgrind warnings / potentially UB
- memset(buffer + values_read, 0, (num_values - values_read) * sizeof(T));
-
- // Add spacing for null entries. As we have filled the buffer from the front,
- // we need to add the spacing from the back.
- int values_to_move = values_read;
- for (int i = num_values - 1; i >= 0; i--) {
- if (::arrow::BitUtil::GetBit(valid_bits, valid_bits_offset + i)) {
- buffer[i] = buffer[--values_to_move];
- }
- }
- return num_values;
- }
-
- // Returns the number of values left (for the last call to SetData()). This is
- // the number of values left in this page.
- int values_left() const { return num_values_; }
-
- Encoding::type encoding() const { return encoding_; }
-
- protected:
- explicit Decoder(const ColumnDescriptor* descr, Encoding::type encoding)
- : descr_(descr), encoding_(encoding), num_values_(0) {}
-
- // For accessing type-specific metadata, like FIXED_LEN_BYTE_ARRAY
- const ColumnDescriptor* descr_;
-
- const Encoding::type encoding_;
- int num_values_;
-};
-
-} // namespace parquet
-
-#endif // PARQUET_ENCODING_H
diff --git a/src/parquet/exception.cc b/src/parquet/exception.cc
deleted file mode 100644
index 5f5525c..0000000
--- a/src/parquet/exception.cc
+++ /dev/null
@@ -1,57 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#include "parquet/exception.h"
-
-#include <exception>
-#include <sstream>
-#include <string>
-
-#include "parquet/util/macros.h"
-
-namespace parquet {
-
-PARQUET_NORETURN void ParquetException::EofException(const std::string& msg) {
- std::stringstream ss;
- ss << "Unexpected end of stream";
- if (!msg.empty()) {
- ss << ": " << msg;
- }
- throw ParquetException(ss.str());
-}
-
-PARQUET_NORETURN void ParquetException::NYI(const std::string& msg) {
- std::stringstream ss;
- ss << "Not yet implemented: " << msg << ".";
- throw ParquetException(ss.str());
-}
-
-PARQUET_NORETURN void ParquetException::Throw(const std::string& msg) {
- throw ParquetException(msg);
-}
-
-ParquetException::ParquetException(const char* msg) : msg_(msg) {}
-
-ParquetException::ParquetException(const std::string& msg) : msg_(msg) {}
-
-ParquetException::ParquetException(const char* msg, std::exception& e) : msg_(msg) {}
-
-ParquetException::~ParquetException() throw() {}
-
-const char* ParquetException::what() const throw() { return msg_.c_str(); }
-
-} // namespace parquet
diff --git a/src/parquet/exception.h b/src/parquet/exception.h
deleted file mode 100644
index 08629be..0000000
--- a/src/parquet/exception.h
+++ /dev/null
@@ -1,79 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#ifndef PARQUET_EXCEPTION_H
-#define PARQUET_EXCEPTION_H
-
-#include <exception>
-#include <sstream>
-#include <string>
-
-#include "arrow/status.h"
-
-#include "parquet/util/macros.h"
-#include "parquet/util/visibility.h"
-
-// PARQUET-1085
-#if !defined(ARROW_UNUSED)
-#define ARROW_UNUSED(x) UNUSED(x)
-#endif
-
-#define PARQUET_CATCH_NOT_OK(s) \
- try { \
- (s); \
- } catch (const ::parquet::ParquetException& e) { \
- return ::arrow::Status::IOError(e.what()); \
- }
-
-#define PARQUET_IGNORE_NOT_OK(s) \
- do { \
- ::arrow::Status _s = (s); \
- ARROW_UNUSED(_s); \
- } while (0)
-
-#define PARQUET_THROW_NOT_OK(s) \
- do { \
- ::arrow::Status _s = (s); \
- if (!_s.ok()) { \
- std::stringstream ss; \
- ss << "Arrow error: " << _s.ToString(); \
- ::parquet::ParquetException::Throw(ss.str()); \
- } \
- } while (0)
-
-namespace parquet {
-
-class PARQUET_EXPORT ParquetException : public std::exception {
- public:
- PARQUET_NORETURN static void EofException(const std::string& msg = "");
- PARQUET_NORETURN static void NYI(const std::string& msg);
- PARQUET_NORETURN static void Throw(const std::string& msg);
-
- explicit ParquetException(const char* msg);
- explicit ParquetException(const std::string& msg);
- explicit ParquetException(const char* msg, exception& e);
-
- virtual ~ParquetException() throw();
- virtual const char* what() const throw();
-
- private:
- std::string msg_;
-};
-
-} // namespace parquet
-
-#endif // PARQUET_EXCEPTION_H
diff --git a/src/parquet/file-deserialize-test.cc b/src/parquet/file-deserialize-test.cc
deleted file mode 100644
index 6b01ac2..0000000
--- a/src/parquet/file-deserialize-test.cc
+++ /dev/null
@@ -1,299 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#include <gtest/gtest.h>
-
-#include <algorithm>
-#include <cstdint>
-#include <cstdlib>
-#include <cstring>
-#include <exception>
-#include <memory>
-#include <string>
-#include <vector>
-
-#include "parquet/column_reader.h"
-#include "parquet/exception.h"
-#include "parquet/file_reader.h"
-#include "parquet/parquet_types.h"
-#include "parquet/thrift.h"
-#include "parquet/types.h"
-#include "parquet/util/memory.h"
-#include "parquet/util/test-common.h"
-
-namespace parquet {
-
-#define ASSERT_OK(expr) \
- do { \
- ::arrow::Status s = (expr); \
- if (!s.ok()) { \
- FAIL() << s.ToString(); \
- } \
- } while (0)
-
-using ::arrow::io::BufferReader;
-
-// Adds page statistics occupying a certain amount of bytes (for testing very
-// large page headers)
-static inline void AddDummyStats(int stat_size, format::DataPageHeader& data_page) {
- std::vector<uint8_t> stat_bytes(stat_size);
- // Some non-zero value
- std::fill(stat_bytes.begin(), stat_bytes.end(), 1);
- data_page.statistics.__set_max(
- std::string(reinterpret_cast<const char*>(stat_bytes.data()), stat_size));
- data_page.__isset.statistics = true;
-}
-
-class TestPageSerde : public ::testing::Test {
- public:
- void SetUp() {
- data_page_header_.encoding = format::Encoding::PLAIN;
- data_page_header_.definition_level_encoding = format::Encoding::RLE;
- data_page_header_.repetition_level_encoding = format::Encoding::RLE;
-
- ResetStream();
- }
-
- void InitSerializedPageReader(int64_t num_rows,
- Compression::type codec = Compression::UNCOMPRESSED) {
- EndStream();
- std::unique_ptr<InputStream> stream;
- stream.reset(new InMemoryInputStream(out_buffer_));
- page_reader_ = PageReader::Open(std::move(stream), num_rows, codec);
- }
-
- void WriteDataPageHeader(int max_serialized_len = 1024, int32_t uncompressed_size = 0,
- int32_t compressed_size = 0) {
- // Simplifying writing serialized data page headers which may or may not
- // have meaningful data associated with them
-
- // Serialize the Page header
- page_header_.__set_data_page_header(data_page_header_);
- page_header_.uncompressed_page_size = uncompressed_size;
- page_header_.compressed_page_size = compressed_size;
- page_header_.type = format::PageType::DATA_PAGE;
-
- ASSERT_NO_THROW(
- SerializeThriftMsg(&page_header_, max_serialized_len, out_stream_.get()));
- }
-
- void ResetStream() { out_stream_.reset(new InMemoryOutputStream); }
-
- void EndStream() { out_buffer_ = out_stream_->GetBuffer(); }
-
- protected:
- std::unique_ptr<InMemoryOutputStream> out_stream_;
- std::shared_ptr<Buffer> out_buffer_;
-
- std::unique_ptr<PageReader> page_reader_;
- format::PageHeader page_header_;
- format::DataPageHeader data_page_header_;
-};
-
-void CheckDataPageHeader(const format::DataPageHeader expected, const Page* page) {
- ASSERT_EQ(PageType::DATA_PAGE, page->type());
-
- const DataPage* data_page = static_cast<const DataPage*>(page);
- ASSERT_EQ(expected.num_values, data_page->num_values());
- ASSERT_EQ(expected.encoding, data_page->encoding());
- ASSERT_EQ(expected.definition_level_encoding, data_page->definition_level_encoding());
- ASSERT_EQ(expected.repetition_level_encoding, data_page->repetition_level_encoding());
-
- if (expected.statistics.__isset.max) {
- ASSERT_EQ(expected.statistics.max, data_page->statistics().max());
- }
- if (expected.statistics.__isset.min) {
- ASSERT_EQ(expected.statistics.min, data_page->statistics().min());
- }
-}
-
-TEST_F(TestPageSerde, DataPage) {
- format::PageHeader out_page_header;
-
- int stats_size = 512;
- const int32_t num_rows = 4444;
- AddDummyStats(stats_size, data_page_header_);
- data_page_header_.num_values = num_rows;
-
- ASSERT_NO_FATAL_FAILURE(WriteDataPageHeader());
- InitSerializedPageReader(num_rows);
- std::shared_ptr<Page> current_page = page_reader_->NextPage();
- ASSERT_NO_FATAL_FAILURE(CheckDataPageHeader(data_page_header_, current_page.get()));
-}
-
-TEST_F(TestPageSerde, TestLargePageHeaders) {
- int stats_size = 256 * 1024; // 256 KB
- AddDummyStats(stats_size, data_page_header_);
-
- // Any number to verify metadata roundtrip
- const int32_t num_rows = 4141;
- data_page_header_.num_values = num_rows;
-
- int max_header_size = 512 * 1024; // 512 KB
- ASSERT_NO_FATAL_FAILURE(WriteDataPageHeader(max_header_size));
- ASSERT_GE(max_header_size, out_stream_->Tell());
-
- // check header size is between 256 KB to 16 MB
- ASSERT_LE(stats_size, out_stream_->Tell());
- ASSERT_GE(kDefaultMaxPageHeaderSize, out_stream_->Tell());
-
- InitSerializedPageReader(num_rows);
- std::shared_ptr<Page> current_page = page_reader_->NextPage();
- ASSERT_NO_FATAL_FAILURE(CheckDataPageHeader(data_page_header_, current_page.get()));
-}
-
-TEST_F(TestPageSerde, TestFailLargePageHeaders) {
- const int32_t num_rows = 1337; // dummy value
-
- int stats_size = 256 * 1024; // 256 KB
- AddDummyStats(stats_size, data_page_header_);
-
- // Serialize the Page header
- int max_header_size = 512 * 1024; // 512 KB
- ASSERT_NO_FATAL_FAILURE(WriteDataPageHeader(max_header_size));
- ASSERT_GE(max_header_size, out_stream_->Tell());
-
- int smaller_max_size = 128 * 1024;
- ASSERT_LE(smaller_max_size, out_stream_->Tell());
- InitSerializedPageReader(num_rows);
-
- // Set the max page header size to 128 KB, which is less than the current
- // header size
- page_reader_->set_max_page_header_size(smaller_max_size);
- ASSERT_THROW(page_reader_->NextPage(), ParquetException);
-}
-
-TEST_F(TestPageSerde, Compression) {
- Compression::type codec_types[5] = {Compression::GZIP, Compression::SNAPPY,
- Compression::BROTLI, Compression::LZ4,
- Compression::ZSTD};
-
- const int32_t num_rows = 32; // dummy value
- data_page_header_.num_values = num_rows;
-
- int num_pages = 10;
-
- std::vector<std::vector<uint8_t>> faux_data;
- faux_data.resize(num_pages);
- for (int i = 0; i < num_pages; ++i) {
- // The pages keep getting larger
- int page_size = (i + 1) * 64;
- test::random_bytes(page_size, 0, &faux_data[i]);
- }
- for (auto codec_type : codec_types) {
- std::unique_ptr<::arrow::Codec> codec = GetCodecFromArrow(codec_type);
-
- std::vector<uint8_t> buffer;
- for (int i = 0; i < num_pages; ++i) {
- const uint8_t* data = faux_data[i].data();
- int data_size = static_cast<int>(faux_data[i].size());
-
- int64_t max_compressed_size = codec->MaxCompressedLen(data_size, data);
- buffer.resize(max_compressed_size);
-
- int64_t actual_size;
- ASSERT_OK(codec->Compress(data_size, data, max_compressed_size, &buffer[0],
- &actual_size));
-
- ASSERT_NO_FATAL_FAILURE(
- WriteDataPageHeader(1024, data_size, static_cast<int32_t>(actual_size)));
- out_stream_->Write(buffer.data(), actual_size);
- }
-
- InitSerializedPageReader(num_rows * num_pages, codec_type);
-
- std::shared_ptr<Page> page;
- const DataPage* data_page;
- for (int i = 0; i < num_pages; ++i) {
- int data_size = static_cast<int>(faux_data[i].size());
- page = page_reader_->NextPage();
- data_page = static_cast<const DataPage*>(page.get());
- ASSERT_EQ(data_size, data_page->size());
- ASSERT_EQ(0, memcmp(faux_data[i].data(), data_page->data(), data_size));
- }
-
- ResetStream();
- }
-}
-
-TEST_F(TestPageSerde, LZONotSupported) {
- // Must await PARQUET-530
- int data_size = 1024;
- std::vector<uint8_t> faux_data(data_size);
- ASSERT_NO_FATAL_FAILURE(WriteDataPageHeader(1024, data_size, data_size));
- out_stream_->Write(faux_data.data(), data_size);
- ASSERT_THROW(InitSerializedPageReader(data_size, Compression::LZO), ParquetException);
-}
-
-// ----------------------------------------------------------------------
-// File structure tests
-
-class TestParquetFileReader : public ::testing::Test {
- public:
- void AssertInvalidFileThrows(const std::shared_ptr<Buffer>& buffer) {
- reader_.reset(new ParquetFileReader());
-
- auto reader = std::make_shared<BufferReader>(buffer);
- auto wrapper = std::unique_ptr<ArrowInputFile>(new ArrowInputFile(reader));
-
- ASSERT_THROW(reader_->Open(ParquetFileReader::Contents::Open(std::move(wrapper))),
- ParquetException);
- }
-
- protected:
- std::unique_ptr<ParquetFileReader> reader_;
-};
-
-TEST_F(TestParquetFileReader, InvalidHeader) {
- const char* bad_header = "PAR2";
-
- auto buffer = std::make_shared<Buffer>(reinterpret_cast<const uint8_t*>(bad_header),
- strlen(bad_header));
- ASSERT_NO_FATAL_FAILURE(AssertInvalidFileThrows(buffer));
-}
-
-TEST_F(TestParquetFileReader, InvalidFooter) {
- // File is smaller than FOOTER_SIZE
- const char* bad_file = "PAR1PAR";
- auto buffer = std::make_shared<Buffer>(reinterpret_cast<const uint8_t*>(bad_file),
- strlen(bad_file));
- ASSERT_NO_FATAL_FAILURE(AssertInvalidFileThrows(buffer));
-
- // Magic number incorrect
- const char* bad_file2 = "PAR1PAR2";
- buffer = std::make_shared<Buffer>(reinterpret_cast<const uint8_t*>(bad_file2),
- strlen(bad_file2));
- ASSERT_NO_FATAL_FAILURE(AssertInvalidFileThrows(buffer));
-}
-
-TEST_F(TestParquetFileReader, IncompleteMetadata) {
- InMemoryOutputStream stream;
-
- const char* magic = "PAR1";
-
- stream.Write(reinterpret_cast<const uint8_t*>(magic), strlen(magic));
- std::vector<uint8_t> bytes(10);
- stream.Write(bytes.data(), bytes.size());
- uint32_t metadata_len = 24;
- stream.Write(reinterpret_cast<const uint8_t*>(&metadata_len), sizeof(uint32_t));
- stream.Write(reinterpret_cast<const uint8_t*>(magic), strlen(magic));
-
- auto buffer = stream.GetBuffer();
- ASSERT_NO_FATAL_FAILURE(AssertInvalidFileThrows(buffer));
-}
-
-} // namespace parquet
diff --git a/src/parquet/file-serialize-test.cc b/src/parquet/file-serialize-test.cc
deleted file mode 100644
index 750faa2..0000000
--- a/src/parquet/file-serialize-test.cc
+++ /dev/null
@@ -1,310 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#include <gtest/gtest.h>
-
-#include "parquet/column_reader.h"
-#include "parquet/column_writer.h"
-#include "parquet/file_reader.h"
-#include "parquet/file_writer.h"
-#include "parquet/test-specialization.h"
-#include "parquet/test-util.h"
-#include "parquet/types.h"
-#include "parquet/util/memory.h"
-
-namespace parquet {
-
-using schema::GroupNode;
-using schema::NodePtr;
-using schema::PrimitiveNode;
-
-namespace test {
-
-template <typename TestType>
-class TestSerialize : public PrimitiveTypedTest<TestType> {
- public:
- typedef typename TestType::c_type T;
-
- void SetUp() {
- num_columns_ = 4;
- num_rowgroups_ = 4;
- rows_per_rowgroup_ = 50;
- rows_per_batch_ = 10;
- this->SetUpSchema(Repetition::OPTIONAL, num_columns_);
- }
-
- protected:
- int num_columns_;
- int num_rowgroups_;
- int rows_per_rowgroup_;
- int rows_per_batch_;
-
- void FileSerializeTest(Compression::type codec_type) {
- std::shared_ptr<InMemoryOutputStream> sink(new InMemoryOutputStream());
- auto gnode = std::static_pointer_cast<GroupNode>(this->node_);
-
- WriterProperties::Builder prop_builder;
-
- for (int i = 0; i < num_columns_; ++i) {
- prop_builder.compression(this->schema_.Column(i)->name(), codec_type);
- }
- std::shared_ptr<WriterProperties> writer_properties = prop_builder.build();
-
- auto file_writer = ParquetFileWriter::Open(sink, gnode, writer_properties);
- this->GenerateData(rows_per_rowgroup_);
- for (int rg = 0; rg < num_rowgroups_ / 2; ++rg) {
- RowGroupWriter* row_group_writer;
- row_group_writer = file_writer->AppendRowGroup();
- for (int col = 0; col < num_columns_; ++col) {
- auto column_writer =
- static_cast<TypedColumnWriter<TestType>*>(row_group_writer->NextColumn());
- column_writer->WriteBatch(rows_per_rowgroup_, this->def_levels_.data(), nullptr,
- this->values_ptr_);
- column_writer->Close();
- // Ensure column() API which is specific to BufferedRowGroup cannot be called
- ASSERT_THROW(row_group_writer->column(col), ParquetException);
- }
-
- row_group_writer->Close();
- }
- // Write half BufferedRowGroups
- for (int rg = 0; rg < num_rowgroups_ / 2; ++rg) {
- RowGroupWriter* row_group_writer;
- row_group_writer = file_writer->AppendBufferedRowGroup();
- for (int batch = 0; batch < (rows_per_rowgroup_ / rows_per_batch_); ++batch) {
- for (int col = 0; col < num_columns_; ++col) {
- auto column_writer =
- static_cast<TypedColumnWriter<TestType>*>(row_group_writer->column(col));
- column_writer->WriteBatch(
- rows_per_batch_, this->def_levels_.data() + (batch * rows_per_batch_),
- nullptr, this->values_ptr_ + (batch * rows_per_batch_));
- // Ensure NextColumn() API which is specific to RowGroup cannot be called
- ASSERT_THROW(row_group_writer->NextColumn(), ParquetException);
- }
- }
- for (int col = 0; col < num_columns_; ++col) {
- auto column_writer =
- static_cast<TypedColumnWriter<TestType>*>(row_group_writer->column(col));
- column_writer->Close();
- }
- row_group_writer->Close();
- }
- file_writer->Close();
-
- auto buffer = sink->GetBuffer();
- int num_rows_ = num_rowgroups_ * rows_per_rowgroup_;
-
- auto source = std::make_shared<::arrow::io::BufferReader>(buffer);
- auto file_reader = ParquetFileReader::Open(source);
- ASSERT_EQ(num_columns_, file_reader->metadata()->num_columns());
- ASSERT_EQ(num_rowgroups_, file_reader->metadata()->num_row_groups());
- ASSERT_EQ(num_rows_, file_reader->metadata()->num_rows());
-
- for (int rg = 0; rg < num_rowgroups_; ++rg) {
- auto rg_reader = file_reader->RowGroup(rg);
- ASSERT_EQ(num_columns_, rg_reader->metadata()->num_columns());
- ASSERT_EQ(rows_per_rowgroup_, rg_reader->metadata()->num_rows());
- // Check that the specified compression was actually used.
- ASSERT_EQ(codec_type, rg_reader->metadata()->ColumnChunk(0)->compression());
-
- int64_t values_read;
-
- for (int i = 0; i < num_columns_; ++i) {
- ASSERT_FALSE(rg_reader->metadata()->ColumnChunk(i)->has_index_page());
- std::vector<int16_t> def_levels_out(rows_per_rowgroup_);
- std::vector<int16_t> rep_levels_out(rows_per_rowgroup_);
- auto col_reader =
- std::static_pointer_cast<TypedColumnReader<TestType>>(rg_reader->Column(i));
- this->SetupValuesOut(rows_per_rowgroup_);
- col_reader->ReadBatch(rows_per_rowgroup_, def_levels_out.data(),
- rep_levels_out.data(), this->values_out_ptr_, &values_read);
- this->SyncValuesOut();
- ASSERT_EQ(rows_per_rowgroup_, values_read);
- ASSERT_EQ(this->values_, this->values_out_);
- ASSERT_EQ(this->def_levels_, def_levels_out);
- }
- }
- }
-
- void UnequalNumRows(int64_t max_rows, const std::vector<int64_t> rows_per_column) {
- std::shared_ptr<InMemoryOutputStream> sink(new InMemoryOutputStream());
- auto gnode = std::static_pointer_cast<GroupNode>(this->node_);
-
- std::shared_ptr<WriterProperties> props = WriterProperties::Builder().build();
-
- auto file_writer = ParquetFileWriter::Open(sink, gnode, props);
-
- RowGroupWriter* row_group_writer;
- row_group_writer = file_writer->AppendRowGroup();
-
- this->GenerateData(max_rows);
- for (int col = 0; col < num_columns_; ++col) {
- auto column_writer =
- static_cast<TypedColumnWriter<TestType>*>(row_group_writer->NextColumn());
- column_writer->WriteBatch(rows_per_column[col], this->def_levels_.data(), nullptr,
- this->values_ptr_);
- column_writer->Close();
- }
- row_group_writer->Close();
- file_writer->Close();
- }
-
- void UnequalNumRowsBuffered(int64_t max_rows,
- const std::vector<int64_t> rows_per_column) {
- std::shared_ptr<InMemoryOutputStream> sink(new InMemoryOutputStream());
- auto gnode = std::static_pointer_cast<GroupNode>(this->node_);
-
- std::shared_ptr<WriterProperties> props = WriterProperties::Builder().build();
-
- auto file_writer = ParquetFileWriter::Open(sink, gnode, props);
-
- RowGroupWriter* row_group_writer;
- row_group_writer = file_writer->AppendBufferedRowGroup();
-
- this->GenerateData(max_rows);
- for (int col = 0; col < num_columns_; ++col) {
- auto column_writer =
- static_cast<TypedColumnWriter<TestType>*>(row_group_writer->column(col));
- column_writer->WriteBatch(rows_per_column[col], this->def_levels_.data(), nullptr,
- this->values_ptr_);
- column_writer->Close();
- }
- row_group_writer->Close();
- file_writer->Close();
- }
-
- void RepeatedUnequalRows() {
- // Optional and repeated, so definition and repetition levels
- this->SetUpSchema(Repetition::REPEATED);
-
- const int kNumRows = 100;
- this->GenerateData(kNumRows);
-
- std::shared_ptr<InMemoryOutputStream> sink(new InMemoryOutputStream());
- auto gnode = std::static_pointer_cast<GroupNode>(this->node_);
- std::shared_ptr<WriterProperties> props = WriterProperties::Builder().build();
- auto file_writer = ParquetFileWriter::Open(sink, gnode, props);
-
- RowGroupWriter* row_group_writer;
- row_group_writer = file_writer->AppendRowGroup();
-
- this->GenerateData(kNumRows);
-
- std::vector<int16_t> definition_levels(kNumRows, 1);
- std::vector<int16_t> repetition_levels(kNumRows, 0);
-
- {
- auto column_writer =
- static_cast<TypedColumnWriter<TestType>*>(row_group_writer->NextColumn());
- column_writer->WriteBatch(kNumRows, definition_levels.data(),
- repetition_levels.data(), this->values_ptr_);
- column_writer->Close();
- }
-
- definition_levels[1] = 0;
- repetition_levels[3] = 1;
-
- {
- auto column_writer =
- static_cast<TypedColumnWriter<TestType>*>(row_group_writer->NextColumn());
- column_writer->WriteBatch(kNumRows, definition_levels.data(),
- repetition_levels.data(), this->values_ptr_);
- column_writer->Close();
- }
- }
-
- void ZeroRowsRowGroup() {
- std::shared_ptr<InMemoryOutputStream> sink(new InMemoryOutputStream());
- auto gnode = std::static_pointer_cast<GroupNode>(this->node_);
-
- std::shared_ptr<WriterProperties> props = WriterProperties::Builder().build();
-
- auto file_writer = ParquetFileWriter::Open(sink, gnode, props);
-
- RowGroupWriter* row_group_writer;
-
- row_group_writer = file_writer->AppendRowGroup();
- for (int col = 0; col < num_columns_; ++col) {
- auto column_writer =
- static_cast<TypedColumnWriter<TestType>*>(row_group_writer->NextColumn());
- column_writer->Close();
- }
- row_group_writer->Close();
-
- row_group_writer = file_writer->AppendBufferedRowGroup();
- for (int col = 0; col < num_columns_; ++col) {
- auto column_writer =
- static_cast<TypedColumnWriter<TestType>*>(row_group_writer->column(col));
- column_writer->Close();
- }
- row_group_writer->Close();
-
- file_writer->Close();
- }
-};
-
-typedef ::testing::Types<Int32Type, Int64Type, Int96Type, FloatType, DoubleType,
- BooleanType, ByteArrayType, FLBAType>
- TestTypes;
-
-TYPED_TEST_CASE(TestSerialize, TestTypes);
-
-TYPED_TEST(TestSerialize, SmallFileUncompressed) {
- ASSERT_NO_FATAL_FAILURE(this->FileSerializeTest(Compression::UNCOMPRESSED));
-}
-
-TYPED_TEST(TestSerialize, TooFewRows) {
- std::vector<int64_t> num_rows = {100, 100, 100, 99};
- ASSERT_THROW(this->UnequalNumRows(100, num_rows), ParquetException);
- ASSERT_THROW(this->UnequalNumRowsBuffered(100, num_rows), ParquetException);
-}
-
-TYPED_TEST(TestSerialize, TooManyRows) {
- std::vector<int64_t> num_rows = {100, 100, 100, 101};
- ASSERT_THROW(this->UnequalNumRows(101, num_rows), ParquetException);
- ASSERT_THROW(this->UnequalNumRowsBuffered(101, num_rows), ParquetException);
-}
-
-TYPED_TEST(TestSerialize, ZeroRows) { ASSERT_NO_THROW(this->ZeroRowsRowGroup()); }
-
-TYPED_TEST(TestSerialize, RepeatedTooFewRows) {
- ASSERT_THROW(this->RepeatedUnequalRows(), ParquetException);
-}
-
-TYPED_TEST(TestSerialize, SmallFileSnappy) {
- ASSERT_NO_FATAL_FAILURE(this->FileSerializeTest(Compression::SNAPPY));
-}
-
-TYPED_TEST(TestSerialize, SmallFileBrotli) {
- ASSERT_NO_FATAL_FAILURE(this->FileSerializeTest(Compression::BROTLI));
-}
-
-TYPED_TEST(TestSerialize, SmallFileGzip) {
- ASSERT_NO_FATAL_FAILURE(this->FileSerializeTest(Compression::GZIP));
-}
-
-TYPED_TEST(TestSerialize, SmallFileLz4) {
- ASSERT_NO_FATAL_FAILURE(this->FileSerializeTest(Compression::LZ4));
-}
-
-TYPED_TEST(TestSerialize, SmallFileZstd) {
- ASSERT_NO_FATAL_FAILURE(this->FileSerializeTest(Compression::ZSTD));
-}
-
-} // namespace test
-
-} // namespace parquet
diff --git a/src/parquet/file_reader.cc b/src/parquet/file_reader.cc
deleted file mode 100644
index c5a0f34..0000000
--- a/src/parquet/file_reader.cc
+++ /dev/null
@@ -1,376 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#include "parquet/file_reader.h"
-
-#include <algorithm>
-#include <cstdint>
-#include <cstdio>
-#include <memory>
-#include <sstream>
-#include <string>
-#include <utility>
-#include <vector>
-
-#include "arrow/io/file.h"
-
-#include "parquet/column_page.h"
-#include "parquet/column_reader.h"
-#include "parquet/column_scanner.h"
-#include "parquet/exception.h"
-#include "parquet/metadata.h"
-#include "parquet/parquet_types.h"
-#include "parquet/properties.h"
-#include "parquet/types.h"
-#include "parquet/util/logging.h"
-#include "parquet/util/memory.h"
-
-using std::string;
-
-namespace arrow {
-
-class Codec;
-
-} // namespace arrow
-
-namespace parquet {
-
-// PARQUET-978: Minimize footer reads by reading 64 KB from the end of the file
-static constexpr int64_t DEFAULT_FOOTER_READ_SIZE = 64 * 1024;
-static constexpr uint32_t FOOTER_SIZE = 8;
-static constexpr uint8_t PARQUET_MAGIC[4] = {'P', 'A', 'R', '1'};
-
-// For PARQUET-816
-static constexpr int64_t kMaxDictHeaderSize = 100;
-
-// ----------------------------------------------------------------------
-// RowGroupReader public API
-
-RowGroupReader::RowGroupReader(std::unique_ptr<Contents> contents)
- : contents_(std::move(contents)) {}
-
-std::shared_ptr<ColumnReader> RowGroupReader::Column(int i) {
- DCHECK(i < metadata()->num_columns())
- << "The RowGroup only has " << metadata()->num_columns()
- << "columns, requested column: " << i;
- const ColumnDescriptor* descr = metadata()->schema()->Column(i);
-
- std::unique_ptr<PageReader> page_reader = contents_->GetColumnPageReader(i);
- return ColumnReader::Make(
- descr, std::move(page_reader),
- const_cast<ReaderProperties*>(contents_->properties())->memory_pool());
-}
-
-std::unique_ptr<PageReader> RowGroupReader::GetColumnPageReader(int i) {
- DCHECK(i < metadata()->num_columns())
- << "The RowGroup only has " << metadata()->num_columns()
- << "columns, requested column: " << i;
- return contents_->GetColumnPageReader(i);
-}
-
-// Returns the rowgroup metadata
-const RowGroupMetaData* RowGroupReader::metadata() const { return contents_->metadata(); }
-
-// RowGroupReader::Contents implementation for the Parquet file specification
-class SerializedRowGroup : public RowGroupReader::Contents {
- public:
- SerializedRowGroup(RandomAccessSource* source, FileMetaData* file_metadata,
- int row_group_number, const ReaderProperties& props)
- : source_(source), file_metadata_(file_metadata), properties_(props) {
- row_group_metadata_ = file_metadata->RowGroup(row_group_number);
- }
-
- const RowGroupMetaData* metadata() const override { return row_group_metadata_.get(); }
-
- const ReaderProperties* properties() const override { return &properties_; }
-
- std::unique_ptr<PageReader> GetColumnPageReader(int i) override {
- // Read column chunk from the file
- auto col = row_group_metadata_->ColumnChunk(i);
-
- int64_t col_start = col->data_page_offset();
- if (col->has_dictionary_page() && col_start > col->dictionary_page_offset()) {
- col_start = col->dictionary_page_offset();
- }
-
- int64_t col_length = col->total_compressed_size();
- std::unique_ptr<InputStream> stream;
-
- // PARQUET-816 workaround for old files created by older parquet-mr
- const ApplicationVersion& version = file_metadata_->writer_version();
- if (version.VersionLt(ApplicationVersion::PARQUET_816_FIXED_VERSION())) {
- // The Parquet MR writer had a bug in 1.2.8 and below where it didn't include the
- // dictionary page header size in total_compressed_size and total_uncompressed_size
- // (see IMPALA-694). We add padding to compensate.
- int64_t bytes_remaining = source_->Size() - (col_start + col_length);
- int64_t padding = std::min<int64_t>(kMaxDictHeaderSize, bytes_remaining);
- col_length += padding;
- }
-
- stream = properties_.GetStream(source_, col_start, col_length);
-
- return PageReader::Open(std::move(stream), col->num_values(), col->compression(),
- properties_.memory_pool());
- }
-
- private:
- RandomAccessSource* source_;
- FileMetaData* file_metadata_;
- std::unique_ptr<RowGroupMetaData> row_group_metadata_;
- ReaderProperties properties_;
-};
-
-// ----------------------------------------------------------------------
-// SerializedFile: An implementation of ParquetFileReader::Contents that deals
-// with the Parquet file structure, Thrift deserialization, and other internal
-// matters
-
-// This class takes ownership of the provided data source
-class SerializedFile : public ParquetFileReader::Contents {
- public:
- SerializedFile(std::unique_ptr<RandomAccessSource> source,
- const ReaderProperties& props = default_reader_properties())
- : source_(std::move(source)), properties_(props) {}
-
- ~SerializedFile() override {
- try {
- Close();
- } catch (...) {
- }
- }
-
- void Close() override { source_->Close(); }
-
- std::shared_ptr<RowGroupReader> GetRowGroup(int i) override {
- std::unique_ptr<SerializedRowGroup> contents(
- new SerializedRowGroup(source_.get(), file_metadata_.get(), i, properties_));
- return std::make_shared<RowGroupReader>(std::move(contents));
- }
-
- std::shared_ptr<FileMetaData> metadata() const override { return file_metadata_; }
-
- void set_metadata(const std::shared_ptr<FileMetaData>& metadata) {
- file_metadata_ = metadata;
- }
-
- void ParseMetaData() {
- int64_t file_size = source_->Size();
-
- if (file_size < FOOTER_SIZE) {
- throw ParquetException("Corrupted file, smaller than file footer");
- }
-
- uint8_t footer_buffer[DEFAULT_FOOTER_READ_SIZE];
- int64_t footer_read_size = std::min(file_size, DEFAULT_FOOTER_READ_SIZE);
- int64_t bytes_read =
- source_->ReadAt(file_size - footer_read_size, footer_read_size, footer_buffer);
-
- // Check if all bytes are read. Check if last 4 bytes read have the magic bits
- if (bytes_read != footer_read_size ||
- memcmp(footer_buffer + footer_read_size - 4, PARQUET_MAGIC, 4) != 0) {
- throw ParquetException("Invalid parquet file. Corrupt footer.");
- }
-
- uint32_t metadata_len =
- *reinterpret_cast<uint32_t*>(footer_buffer + footer_read_size - FOOTER_SIZE);
- int64_t metadata_start = file_size - FOOTER_SIZE - metadata_len;
- if (FOOTER_SIZE + metadata_len > file_size) {
- throw ParquetException(
- "Invalid parquet file. File is less than "
- "file metadata size.");
- }
-
- std::shared_ptr<ResizableBuffer> metadata_buffer =
- AllocateBuffer(properties_.memory_pool(), metadata_len);
-
- // Check if the footer_buffer contains the entire metadata
- if (footer_read_size >= (metadata_len + FOOTER_SIZE)) {
- memcpy(metadata_buffer->mutable_data(),
- footer_buffer + (footer_read_size - metadata_len - FOOTER_SIZE),
- metadata_len);
- } else {
- bytes_read =
- source_->ReadAt(metadata_start, metadata_len, metadata_buffer->mutable_data());
- if (bytes_read != metadata_len) {
- throw ParquetException("Invalid parquet file. Could not read metadata bytes.");
- }
- }
-
- file_metadata_ = FileMetaData::Make(metadata_buffer->data(), &metadata_len);
- }
-
- private:
- std::unique_ptr<RandomAccessSource> source_;
- std::shared_ptr<FileMetaData> file_metadata_;
- ReaderProperties properties_;
-};
-
-// ----------------------------------------------------------------------
-// ParquetFileReader public API
-
-ParquetFileReader::ParquetFileReader() {}
-
-ParquetFileReader::~ParquetFileReader() {
- try {
- Close();
- } catch (...) {
- }
-}
-
-// Open the file. If no metadata is passed, it is parsed from the footer of
-// the file
-std::unique_ptr<ParquetFileReader::Contents> ParquetFileReader::Contents::Open(
- std::unique_ptr<RandomAccessSource> source, const ReaderProperties& props,
- const std::shared_ptr<FileMetaData>& metadata) {
- std::unique_ptr<ParquetFileReader::Contents> result(
- new SerializedFile(std::move(source), props));
-
- // Access private methods here, but otherwise unavailable
- SerializedFile* file = static_cast<SerializedFile*>(result.get());
-
- if (metadata == nullptr) {
- // Validates magic bytes, parses metadata, and initializes the SchemaDescriptor
- file->ParseMetaData();
- } else {
- file->set_metadata(metadata);
- }
-
- return result;
-}
-
-std::unique_ptr<ParquetFileReader> ParquetFileReader::Open(
- const std::shared_ptr<::arrow::io::ReadableFileInterface>& source,
- const ReaderProperties& props, const std::shared_ptr<FileMetaData>& metadata) {
- std::unique_ptr<RandomAccessSource> io_wrapper(new ArrowInputFile(source));
- return Open(std::move(io_wrapper), props, metadata);
-}
-
-std::unique_ptr<ParquetFileReader> ParquetFileReader::Open(
- std::unique_ptr<RandomAccessSource> source, const ReaderProperties& props,
- const std::shared_ptr<FileMetaData>& metadata) {
- auto contents = SerializedFile::Open(std::move(source), props, metadata);
- std::unique_ptr<ParquetFileReader> result(new ParquetFileReader());
- result->Open(std::move(contents));
- return result;
-}
-
-std::unique_ptr<ParquetFileReader> ParquetFileReader::OpenFile(
- const std::string& path, bool memory_map, const ReaderProperties& props,
- const std::shared_ptr<FileMetaData>& metadata) {
- std::shared_ptr<::arrow::io::ReadableFileInterface> source;
- if (memory_map) {
- std::shared_ptr<::arrow::io::MemoryMappedFile> handle;
- PARQUET_THROW_NOT_OK(
- ::arrow::io::MemoryMappedFile::Open(path, ::arrow::io::FileMode::READ, &handle));
- source = handle;
- } else {
- std::shared_ptr<::arrow::io::ReadableFile> handle;
- PARQUET_THROW_NOT_OK(
- ::arrow::io::ReadableFile::Open(path, props.memory_pool(), &handle));
- source = handle;
- }
-
- return Open(source, props, metadata);
-}
-
-void ParquetFileReader::Open(std::unique_ptr<ParquetFileReader::Contents> contents) {
- contents_ = std::move(contents);
-}
-
-void ParquetFileReader::Close() {
- if (contents_) {
- contents_->Close();
- }
-}
-
-std::shared_ptr<FileMetaData> ParquetFileReader::metadata() const {
- return contents_->metadata();
-}
-
-std::shared_ptr<RowGroupReader> ParquetFileReader::RowGroup(int i) {
- DCHECK(i < metadata()->num_row_groups())
- << "The file only has " << metadata()->num_row_groups()
- << "row groups, requested reader for: " << i;
- return contents_->GetRowGroup(i);
-}
-
-// ----------------------------------------------------------------------
-// File metadata helpers
-
-std::shared_ptr<FileMetaData> ReadMetaData(
- const std::shared_ptr<::arrow::io::ReadableFileInterface>& source) {
- return ParquetFileReader::Open(source)->metadata();
-}
-
-// ----------------------------------------------------------------------
-// File scanner for performance testing
-
-int64_t ScanFileContents(std::vector<int> columns, const int32_t column_batch_size,
- ParquetFileReader* reader) {
- std::vector<int16_t> rep_levels(column_batch_size);
- std::vector<int16_t> def_levels(column_batch_size);
-
- int num_columns = static_cast<int>(columns.size());
-
- // columns are not specified explicitly. Add all columns
- if (columns.size() == 0) {
- num_columns = reader->metadata()->num_columns();
- columns.resize(num_columns);
- for (int i = 0; i < num_columns; i++) {
- columns[i] = i;
- }
- }
-
- std::vector<int64_t> total_rows(num_columns, 0);
-
- for (int r = 0; r < reader->metadata()->num_row_groups(); ++r) {
- auto group_reader = reader->RowGroup(r);
- int col = 0;
- for (auto i : columns) {
- std::shared_ptr<ColumnReader> col_reader = group_reader->Column(i);
- size_t value_byte_size = GetTypeByteSize(col_reader->descr()->physical_type());
- std::vector<uint8_t> values(column_batch_size * value_byte_size);
-
- int64_t values_read = 0;
- while (col_reader->HasNext()) {
- int64_t levels_read =
- ScanAllValues(column_batch_size, def_levels.data(), rep_levels.data(),
- values.data(), &values_read, col_reader.get());
- if (col_reader->descr()->max_repetition_level() > 0) {
- for (int64_t i = 0; i < levels_read; i++) {
- if (rep_levels[i] == 0) {
- total_rows[col]++;
- }
- }
- } else {
- total_rows[col] += levels_read;
- }
- }
- col++;
- }
- }
-
- for (int i = 1; i < num_columns; ++i) {
- if (total_rows[0] != total_rows[i]) {
- throw ParquetException("Parquet error: Total rows among columns do not match");
- }
- }
-
- return total_rows[0];
-}
-
-} // namespace parquet
diff --git a/src/parquet/file_reader.h b/src/parquet/file_reader.h
deleted file mode 100644
index f751e9b..0000000
--- a/src/parquet/file_reader.h
+++ /dev/null
@@ -1,142 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#ifndef PARQUET_FILE_READER_H
-#define PARQUET_FILE_READER_H
-
-#include <cstdint>
-#include <iosfwd>
-#include <list>
-#include <memory>
-#include <string>
-#include <vector>
-
-#include "parquet/column_reader.h"
-#include "parquet/metadata.h"
-#include "parquet/properties.h"
-#include "parquet/schema.h"
-#include "parquet/statistics.h"
-#include "parquet/util/memory.h"
-#include "parquet/util/visibility.h"
-
-namespace parquet {
-
-class ColumnReader;
-
-class PARQUET_EXPORT RowGroupReader {
- public:
- // Forward declare a virtual class 'Contents' to aid dependency injection and more
- // easily create test fixtures
- // An implementation of the Contents class is defined in the .cc file
- struct Contents {
- virtual ~Contents() {}
- virtual std::unique_ptr<PageReader> GetColumnPageReader(int i) = 0;
- virtual const RowGroupMetaData* metadata() const = 0;
- virtual const ReaderProperties* properties() const = 0;
- };
-
- explicit RowGroupReader(std::unique_ptr<Contents> contents);
-
- // Returns the rowgroup metadata
- const RowGroupMetaData* metadata() const;
-
- // Construct a ColumnReader for the indicated row group-relative
- // column. Ownership is shared with the RowGroupReader.
- std::shared_ptr<ColumnReader> Column(int i);
-
- std::unique_ptr<PageReader> GetColumnPageReader(int i);
-
- private:
- // Holds a pointer to an instance of Contents implementation
- std::unique_ptr<Contents> contents_;
-};
-
-class PARQUET_EXPORT ParquetFileReader {
- public:
- // Forward declare a virtual class 'Contents' to aid dependency injection and more
- // easily create test fixtures
- // An implementation of the Contents class is defined in the .cc file
- struct Contents {
- static std::unique_ptr<Contents> Open(
- std::unique_ptr<RandomAccessSource> source,
- const ReaderProperties& props = default_reader_properties(),
- const std::shared_ptr<FileMetaData>& metadata = nullptr);
-
- virtual ~Contents() {}
- // Perform any cleanup associated with the file contents
- virtual void Close() = 0;
- virtual std::shared_ptr<RowGroupReader> GetRowGroup(int i) = 0;
- virtual std::shared_ptr<FileMetaData> metadata() const = 0;
- };
-
- ParquetFileReader();
- ~ParquetFileReader();
-
- // Create a reader from some implementation of parquet-cpp's generic file
- // input interface
- //
- // If you cannot provide exclusive access to your file resource, create a
- // subclass of RandomAccessSource that wraps the shared resource
- static std::unique_ptr<ParquetFileReader> Open(
- std::unique_ptr<RandomAccessSource> source,
- const ReaderProperties& props = default_reader_properties(),
- const std::shared_ptr<FileMetaData>& metadata = nullptr);
-
- // Create a file reader instance from an Arrow file object. Thread-safety is
- // the responsibility of the file implementation
- static std::unique_ptr<ParquetFileReader> Open(
- const std::shared_ptr<::arrow::io::ReadableFileInterface>& source,
- const ReaderProperties& props = default_reader_properties(),
- const std::shared_ptr<FileMetaData>& metadata = nullptr);
-
- // API Convenience to open a serialized Parquet file on disk, using Arrow IO
- // interfaces.
- static std::unique_ptr<ParquetFileReader> OpenFile(
- const std::string& path, bool memory_map = true,
- const ReaderProperties& props = default_reader_properties(),
- const std::shared_ptr<FileMetaData>& metadata = nullptr);
-
- void Open(std::unique_ptr<Contents> contents);
- void Close();
-
- // The RowGroupReader is owned by the FileReader
- std::shared_ptr<RowGroupReader> RowGroup(int i);
-
- // Returns the file metadata. Only one instance is ever created
- std::shared_ptr<FileMetaData> metadata() const;
-
- private:
- // Holds a pointer to an instance of Contents implementation
- std::unique_ptr<Contents> contents_;
-};
-
-// Read only Parquet file metadata
-std::shared_ptr<FileMetaData> PARQUET_EXPORT
-ReadMetaData(const std::shared_ptr<::arrow::io::ReadableFileInterface>& source);
-
-/// \brief Scan all values in file. Useful for performance testing
-/// \param[in] columns the column numbers to scan. If empty scans all
-/// \param[in] column_batch_size number of values to read at a time when scanning column
-/// \param[in] reader a ParquetFileReader instance
-/// \return number of semantic rows in file
-PARQUET_EXPORT
-int64_t ScanFileContents(std::vector<int> columns, const int32_t column_batch_size,
- ParquetFileReader* reader);
-
-} // namespace parquet
-
-#endif // PARQUET_FILE_READER_H
diff --git a/src/parquet/file_writer.cc b/src/parquet/file_writer.cc
deleted file mode 100644
index 30673c5..0000000
--- a/src/parquet/file_writer.cc
+++ /dev/null
@@ -1,419 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#include "parquet/file_writer.h"
-
-#include <vector>
-
-#include "parquet/column_writer.h"
-#include "parquet/schema-internal.h"
-#include "parquet/schema.h"
-#include "parquet/thrift.h"
-#include "parquet/util/memory.h"
-
-using arrow::MemoryPool;
-
-using parquet::schema::GroupNode;
-using parquet::schema::SchemaFlattener;
-
-namespace parquet {
-
-// FIXME: copied from reader-internal.cc
-static constexpr uint8_t PARQUET_MAGIC[4] = {'P', 'A', 'R', '1'};
-
-// ----------------------------------------------------------------------
-// RowGroupWriter public API
-
-RowGroupWriter::RowGroupWriter(std::unique_ptr<Contents> contents)
- : contents_(std::move(contents)) {}
-
-void RowGroupWriter::Close() {
- if (contents_) {
- contents_->Close();
- }
-}
-
-ColumnWriter* RowGroupWriter::NextColumn() { return contents_->NextColumn(); }
-
-ColumnWriter* RowGroupWriter::column(int i) { return contents_->column(i); }
-
-int64_t RowGroupWriter::total_compressed_bytes() const {
- return contents_->total_compressed_bytes();
-}
-
-int64_t RowGroupWriter::total_bytes_written() const {
- return contents_->total_bytes_written();
-}
-
-int RowGroupWriter::current_column() { return contents_->current_column(); }
-
-int RowGroupWriter::num_columns() const { return contents_->num_columns(); }
-
-int64_t RowGroupWriter::num_rows() const { return contents_->num_rows(); }
-
-inline void ThrowRowsMisMatchError(int col, int64_t prev, int64_t curr) {
- std::stringstream ss;
- ss << "Column " << col << " had " << curr << " while previous column had " << prev;
- throw ParquetException(ss.str());
-}
-
-// ----------------------------------------------------------------------
-// RowGroupSerializer
-
-// RowGroupWriter::Contents implementation for the Parquet file specification
-class RowGroupSerializer : public RowGroupWriter::Contents {
- public:
- RowGroupSerializer(OutputStream* sink, RowGroupMetaDataBuilder* metadata,
- const WriterProperties* properties, bool buffered_row_group = false)
- : sink_(sink),
- metadata_(metadata),
- properties_(properties),
- total_bytes_written_(0),
- closed_(false),
- current_column_index_(0),
- num_rows_(0),
- buffered_row_group_(buffered_row_group) {
- if (buffered_row_group) {
- InitColumns();
- } else {
- column_writers_.push_back(nullptr);
- }
- }
-
- int num_columns() const override { return metadata_->num_columns(); }
-
- int64_t num_rows() const override {
- CheckRowsWritten();
- // CheckRowsWritten ensures num_rows_ is set correctly
- return num_rows_;
- }
-
- ColumnWriter* NextColumn() override {
- if (buffered_row_group_) {
- throw ParquetException(
- "NextColumn() is not supported when a RowGroup is written by size");
- }
-
- if (column_writers_[0]) {
- CheckRowsWritten();
- }
-
- // Throws an error if more columns are being written
- auto col_meta = metadata_->NextColumnChunk();
-
- if (column_writers_[0]) {
- total_bytes_written_ += column_writers_[0]->Close();
- }
-
- ++current_column_index_;
-
- const ColumnDescriptor* column_descr = col_meta->descr();
- std::unique_ptr<PageWriter> pager =
- PageWriter::Open(sink_, properties_->compression(column_descr->path()), col_meta,
- properties_->memory_pool());
- column_writers_[0] = ColumnWriter::Make(col_meta, std::move(pager), properties_);
- return column_writers_[0].get();
- }
-
- ColumnWriter* column(int i) override {
- if (!buffered_row_group_) {
- throw ParquetException(
- "column() is only supported when a BufferedRowGroup is being written");
- }
-
- if (i >= 0 && i < static_cast<int>(column_writers_.size())) {
- return column_writers_[i].get();
- }
- return nullptr;
- }
-
- int current_column() const override { return metadata_->current_column(); }
-
- int64_t total_compressed_bytes() const override {
- int64_t total_compressed_bytes = 0;
- for (size_t i = 0; i < column_writers_.size(); i++) {
- if (column_writers_[i]) {
- total_compressed_bytes += column_writers_[i]->total_compressed_bytes();
- }
- }
- return total_compressed_bytes;
- }
-
- int64_t total_bytes_written() const override {
- int64_t total_bytes_written = 0;
- for (size_t i = 0; i < column_writers_.size(); i++) {
- if (column_writers_[i]) {
- total_bytes_written += column_writers_[i]->total_bytes_written();
- }
- }
- return total_bytes_written;
- }
-
- void Close() override {
- if (!closed_) {
- closed_ = true;
- CheckRowsWritten();
-
- for (size_t i = 0; i < column_writers_.size(); i++) {
- if (column_writers_[i]) {
- total_bytes_written_ += column_writers_[i]->Close();
- column_writers_[i].reset();
- }
- }
-
- column_writers_.clear();
-
- // Ensures all columns have been written
- metadata_->set_num_rows(num_rows_);
- metadata_->Finish(total_bytes_written_);
- }
- }
-
- private:
- OutputStream* sink_;
- mutable RowGroupMetaDataBuilder* metadata_;
- const WriterProperties* properties_;
- int64_t total_bytes_written_;
- bool closed_;
- int current_column_index_;
- mutable int64_t num_rows_;
- bool buffered_row_group_;
-
- void CheckRowsWritten() const {
- // verify when only one column is written at a time
- if (!buffered_row_group_ && column_writers_.size() > 0 && column_writers_[0]) {
- int64_t current_col_rows = column_writers_[0]->rows_written();
- if (num_rows_ == 0) {
- num_rows_ = current_col_rows;
- } else if (num_rows_ != current_col_rows) {
- ThrowRowsMisMatchError(current_column_index_, current_col_rows, num_rows_);
- }
- } else if (buffered_row_group_ &&
- column_writers_.size() > 0) { // when buffered_row_group = true
- int64_t current_col_rows = column_writers_[0]->rows_written();
- for (int i = 1; i < static_cast<int>(column_writers_.size()); i++) {
- int64_t current_col_rows_i = column_writers_[i]->rows_written();
- if (current_col_rows != current_col_rows_i) {
- ThrowRowsMisMatchError(i, current_col_rows_i, current_col_rows);
- }
- }
- num_rows_ = current_col_rows;
- }
- }
-
- void InitColumns() {
- for (int i = 0; i < num_columns(); i++) {
- auto col_meta = metadata_->NextColumnChunk();
- const ColumnDescriptor* column_descr = col_meta->descr();
- std::unique_ptr<PageWriter> pager =
- PageWriter::Open(sink_, properties_->compression(column_descr->path()),
- col_meta, properties_->memory_pool(), buffered_row_group_);
- column_writers_.push_back(
- ColumnWriter::Make(col_meta, std::move(pager), properties_));
- }
- }
-
- std::vector<std::shared_ptr<ColumnWriter>> column_writers_;
-};
-
-// ----------------------------------------------------------------------
-// FileSerializer
-
-// An implementation of ParquetFileWriter::Contents that deals with the Parquet
-// file structure, Thrift serialization, and other internal matters
-
-class FileSerializer : public ParquetFileWriter::Contents {
- public:
- static std::unique_ptr<ParquetFileWriter::Contents> Open(
- const std::shared_ptr<OutputStream>& sink, const std::shared_ptr<GroupNode>& schema,
- const std::shared_ptr<WriterProperties>& properties,
- const std::shared_ptr<const KeyValueMetadata>& key_value_metadata) {
- std::unique_ptr<ParquetFileWriter::Contents> result(
- new FileSerializer(sink, schema, properties, key_value_metadata));
-
- return result;
- }
-
- void Close() override {
- if (is_open_) {
- if (row_group_writer_) {
- num_rows_ += row_group_writer_->num_rows();
- row_group_writer_->Close();
- }
- row_group_writer_.reset();
-
- // Write magic bytes and metadata
- auto metadata = metadata_->Finish();
- WriteFileMetaData(*metadata, sink_.get());
-
- sink_->Close();
- is_open_ = false;
- }
- }
-
- int num_columns() const override { return schema_.num_columns(); }
-
- int num_row_groups() const override { return num_row_groups_; }
-
- int64_t num_rows() const override { return num_rows_; }
-
- const std::shared_ptr<WriterProperties>& properties() const override {
- return properties_;
- }
-
- RowGroupWriter* AppendRowGroup(bool buffered_row_group) {
- if (row_group_writer_) {
- row_group_writer_->Close();
- }
- num_row_groups_++;
- auto rg_metadata = metadata_->AppendRowGroup();
- std::unique_ptr<RowGroupWriter::Contents> contents(new RowGroupSerializer(
- sink_.get(), rg_metadata, properties_.get(), buffered_row_group));
- row_group_writer_.reset(new RowGroupWriter(std::move(contents)));
- return row_group_writer_.get();
- }
-
- RowGroupWriter* AppendRowGroup() override { return AppendRowGroup(false); }
-
- RowGroupWriter* AppendBufferedRowGroup() override { return AppendRowGroup(true); }
-
- ~FileSerializer() override {
- try {
- Close();
- } catch (...) {
- }
- }
-
- private:
- FileSerializer(const std::shared_ptr<OutputStream>& sink,
- const std::shared_ptr<GroupNode>& schema,
- const std::shared_ptr<WriterProperties>& properties,
- const std::shared_ptr<const KeyValueMetadata>& key_value_metadata)
- : ParquetFileWriter::Contents(schema, key_value_metadata),
- sink_(sink),
- is_open_(true),
- properties_(properties),
- num_row_groups_(0),
- num_rows_(0),
- metadata_(FileMetaDataBuilder::Make(&schema_, properties, key_value_metadata)) {
- StartFile();
- }
-
- std::shared_ptr<OutputStream> sink_;
- bool is_open_;
- const std::shared_ptr<WriterProperties> properties_;
- int num_row_groups_;
- int64_t num_rows_;
- std::unique_ptr<FileMetaDataBuilder> metadata_;
- // Only one of the row group writers is active at a time
- std::unique_ptr<RowGroupWriter> row_group_writer_;
-
- void StartFile() {
- // Parquet files always start with PAR1
- sink_->Write(PARQUET_MAGIC, 4);
- }
-};
-
-// ----------------------------------------------------------------------
-// ParquetFileWriter public API
-
-ParquetFileWriter::ParquetFileWriter() {}
-
-ParquetFileWriter::~ParquetFileWriter() {
- try {
- Close();
- } catch (...) {
- }
-}
-
-std::unique_ptr<ParquetFileWriter> ParquetFileWriter::Open(
- const std::shared_ptr<::arrow::io::OutputStream>& sink,
- const std::shared_ptr<GroupNode>& schema,
- const std::shared_ptr<WriterProperties>& properties,
- const std::shared_ptr<const KeyValueMetadata>& key_value_metadata) {
- return Open(std::make_shared<ArrowOutputStream>(sink), schema, properties,
- key_value_metadata);
-}
-
-std::unique_ptr<ParquetFileWriter> ParquetFileWriter::Open(
- const std::shared_ptr<OutputStream>& sink,
- const std::shared_ptr<schema::GroupNode>& schema,
- const std::shared_ptr<WriterProperties>& properties,
- const std::shared_ptr<const KeyValueMetadata>& key_value_metadata) {
- auto contents = FileSerializer::Open(sink, schema, properties, key_value_metadata);
- std::unique_ptr<ParquetFileWriter> result(new ParquetFileWriter());
- result->Open(std::move(contents));
- return result;
-}
-
-void WriteFileMetaData(const FileMetaData& file_metadata, OutputStream* sink) {
- // Write MetaData
- uint32_t metadata_len = static_cast<uint32_t>(sink->Tell());
-
- file_metadata.WriteTo(sink);
- metadata_len = static_cast<uint32_t>(sink->Tell()) - metadata_len;
-
- // Write Footer
- sink->Write(reinterpret_cast<uint8_t*>(&metadata_len), 4);
- sink->Write(PARQUET_MAGIC, 4);
-}
-
-const SchemaDescriptor* ParquetFileWriter::schema() const { return contents_->schema(); }
-
-const ColumnDescriptor* ParquetFileWriter::descr(int i) const {
- return contents_->schema()->Column(i);
-}
-
-int ParquetFileWriter::num_columns() const { return contents_->num_columns(); }
-
-int64_t ParquetFileWriter::num_rows() const { return contents_->num_rows(); }
-
-int ParquetFileWriter::num_row_groups() const { return contents_->num_row_groups(); }
-
-const std::shared_ptr<const KeyValueMetadata>& ParquetFileWriter::key_value_metadata()
- const {
- return contents_->key_value_metadata();
-}
-
-void ParquetFileWriter::Open(std::unique_ptr<ParquetFileWriter::Contents> contents) {
- contents_ = std::move(contents);
-}
-
-void ParquetFileWriter::Close() {
- if (contents_) {
- contents_->Close();
- contents_.reset();
- }
-}
-
-RowGroupWriter* ParquetFileWriter::AppendRowGroup() {
- return contents_->AppendRowGroup();
-}
-
-RowGroupWriter* ParquetFileWriter::AppendBufferedRowGroup() {
- return contents_->AppendBufferedRowGroup();
-}
-
-RowGroupWriter* ParquetFileWriter::AppendRowGroup(int64_t num_rows) {
- return AppendRowGroup();
-}
-
-const std::shared_ptr<WriterProperties>& ParquetFileWriter::properties() const {
- return contents_->properties();
-}
-
-} // namespace parquet
diff --git a/src/parquet/file_writer.h b/src/parquet/file_writer.h
deleted file mode 100644
index cdfe06c..0000000
--- a/src/parquet/file_writer.h
+++ /dev/null
@@ -1,219 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#ifndef PARQUET_FILE_WRITER_H
-#define PARQUET_FILE_WRITER_H
-
-#include <cstdint>
-#include <memory>
-
-#include "parquet/metadata.h"
-#include "parquet/properties.h"
-#include "parquet/schema.h"
-#include "parquet/util/memory.h"
-#include "parquet/util/visibility.h"
-
-namespace parquet {
-
-class ColumnWriter;
-class PageWriter;
-class OutputStream;
-
-namespace schema {
-
-class GroupNode;
-
-} // namespace schema
-
-class PARQUET_EXPORT RowGroupWriter {
- public:
- // Forward declare a virtual class 'Contents' to aid dependency injection and more
- // easily create test fixtures
- // An implementation of the Contents class is defined in the .cc file
- struct Contents {
- virtual ~Contents() = default;
- virtual int num_columns() const = 0;
- virtual int64_t num_rows() const = 0;
-
- // to be used only with ParquetFileWriter::AppendRowGroup
- virtual ColumnWriter* NextColumn() = 0;
- // to be used only with ParquetFileWriter::AppendBufferedRowGroup
- virtual ColumnWriter* column(int i) = 0;
-
- virtual int current_column() const = 0;
- virtual void Close() = 0;
-
- // total bytes written by the page writer
- virtual int64_t total_bytes_written() const = 0;
- // total bytes still compressed but not written
- virtual int64_t total_compressed_bytes() const = 0;
- };
-
- explicit RowGroupWriter(std::unique_ptr<Contents> contents);
-
- /// Construct a ColumnWriter for the indicated row group-relative column.
- ///
- /// To be used only with ParquetFileWriter::AppendRowGroup
- /// Ownership is solely within the RowGroupWriter. The ColumnWriter is only
- /// valid until the next call to NextColumn or Close. As the contents are
- /// directly written to the sink, once a new column is started, the contents
- /// of the previous one cannot be modified anymore.
- ColumnWriter* NextColumn();
- /// Index of currently written column
- int current_column();
- void Close();
-
- int num_columns() const;
-
- /// Construct a ColumnWriter for the indicated row group column.
- ///
- /// To be used only with ParquetFileWriter::AppendBufferedRowGroup
- /// Ownership is solely within the RowGroupWriter. The ColumnWriter is
- /// valid until Close. The contents are buffered in memory and written to sink
- /// on Close
- ColumnWriter* column(int i);
-
- /**
- * Number of rows that shall be written as part of this RowGroup.
- */
- int64_t num_rows() const;
-
- int64_t total_bytes_written() const;
- int64_t total_compressed_bytes() const;
-
- private:
- // Holds a pointer to an instance of Contents implementation
- std::unique_ptr<Contents> contents_;
-};
-
-PARQUET_EXPORT
-void WriteFileMetaData(const FileMetaData& file_metadata, OutputStream* sink);
-
-class PARQUET_EXPORT ParquetFileWriter {
- public:
- // Forward declare a virtual class 'Contents' to aid dependency injection and more
- // easily create test fixtures
- // An implementation of the Contents class is defined in the .cc file
- struct Contents {
- Contents(const std::shared_ptr<::parquet::schema::GroupNode>& schema,
- const std::shared_ptr<const KeyValueMetadata>& key_value_metadata)
- : schema_(), key_value_metadata_(key_value_metadata) {
- schema_.Init(schema);
- }
- virtual ~Contents() {}
- // Perform any cleanup associated with the file contents
- virtual void Close() = 0;
-
- /// \note Deprecated since 1.3.0
- RowGroupWriter* AppendRowGroup(int64_t num_rows);
-
- virtual RowGroupWriter* AppendRowGroup() = 0;
- virtual RowGroupWriter* AppendBufferedRowGroup() = 0;
-
- virtual int64_t num_rows() const = 0;
- virtual int num_columns() const = 0;
- virtual int num_row_groups() const = 0;
-
- virtual const std::shared_ptr<WriterProperties>& properties() const = 0;
-
- const std::shared_ptr<const KeyValueMetadata>& key_value_metadata() const {
- return key_value_metadata_;
- }
-
- // Return const-pointer to make it clear that this object is not to be copied
- const SchemaDescriptor* schema() const { return &schema_; }
-
- SchemaDescriptor schema_;
-
- /// This should be the only place this is stored. Everything else is a const reference
- std::shared_ptr<const KeyValueMetadata> key_value_metadata_;
- };
-
- ParquetFileWriter();
- ~ParquetFileWriter();
-
- static std::unique_ptr<ParquetFileWriter> Open(
- const std::shared_ptr<::arrow::io::OutputStream>& sink,
- const std::shared_ptr<schema::GroupNode>& schema,
- const std::shared_ptr<WriterProperties>& properties = default_writer_properties(),
- const std::shared_ptr<const KeyValueMetadata>& key_value_metadata = nullptr);
-
- static std::unique_ptr<ParquetFileWriter> Open(
- const std::shared_ptr<OutputStream>& sink,
- const std::shared_ptr<schema::GroupNode>& schema,
- const std::shared_ptr<WriterProperties>& properties = default_writer_properties(),
- const std::shared_ptr<const KeyValueMetadata>& key_value_metadata = nullptr);
-
- void Open(std::unique_ptr<Contents> contents);
- void Close();
-
- // Construct a RowGroupWriter for the indicated number of rows.
- //
- // Ownership is solely within the ParquetFileWriter. The RowGroupWriter is only valid
- // until the next call to AppendRowGroup or AppendBufferedRowGroup or Close.
- // @param num_rows The number of rows that are stored in the new RowGroup
- //
- // \deprecated Since 1.3.0
- RowGroupWriter* AppendRowGroup(int64_t num_rows);
-
- /// Construct a RowGroupWriter with an arbitrary number of rows.
- ///
- /// Ownership is solely within the ParquetFileWriter. The RowGroupWriter is only valid
- /// until the next call to AppendRowGroup or AppendBufferedRowGroup or Close.
- RowGroupWriter* AppendRowGroup();
-
- /// Construct a RowGroupWriter that buffers all the values until the RowGroup is ready.
- /// Use this if you want to write a RowGroup based on a certain size
- ///
- /// Ownership is solely within the ParquetFileWriter. The RowGroupWriter is only valid
- /// until the next call to AppendRowGroup or AppendBufferedRowGroup or Close.
- RowGroupWriter* AppendBufferedRowGroup();
-
- /// Number of columns.
- ///
- /// This number is fixed during the lifetime of the writer as it is determined via
- /// the schema.
- int num_columns() const;
-
- /// Number of rows in the yet started RowGroups.
- ///
- /// Changes on the addition of a new RowGroup.
- int64_t num_rows() const;
-
- /// Number of started RowGroups.
- int num_row_groups() const;
-
- /// Configuration passed to the writer, e.g. the used Parquet format version.
- const std::shared_ptr<WriterProperties>& properties() const;
-
- /// Returns the file schema descriptor
- const SchemaDescriptor* schema() const;
-
- /// Returns a column descriptor in schema
- const ColumnDescriptor* descr(int i) const;
-
- /// Returns the file custom metadata
- const std::shared_ptr<const KeyValueMetadata>& key_value_metadata() const;
-
- private:
- // Holds a pointer to an instance of Contents implementation
- std::unique_ptr<Contents> contents_;
-};
-
-} // namespace parquet
-
-#endif // PARQUET_FILE_WRITER_H
diff --git a/src/parquet/hasher.h b/src/parquet/hasher.h
deleted file mode 100644
index dc316a0..0000000
--- a/src/parquet/hasher.h
+++ /dev/null
@@ -1,75 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#ifndef PARQUET_HASHER_H
-#define PARQUET_HASHER_H
-
-#include <cstdint>
-#include "parquet/types.h"
-
-namespace parquet {
-// Abstract class for hash
-class Hasher {
- public:
- /// Compute hash for 32 bits value by using its plain encoding result.
- ///
- /// @param value the value to hash.
- /// @return hash result.
- virtual uint64_t Hash(int32_t value) const = 0;
-
- /// Compute hash for 64 bits value by using its plain encoding result.
- ///
- /// @param value the value to hash.
- /// @return hash result.
- virtual uint64_t Hash(int64_t value) const = 0;
-
- /// Compute hash for float value by using its plain encoding result.
- ///
- /// @param value the value to hash.
- /// @return hash result.
- virtual uint64_t Hash(float value) const = 0;
-
- /// Compute hash for double value by using its plain encoding result.
- ///
- /// @param value the value to hash.
- /// @return hash result.
- virtual uint64_t Hash(double value) const = 0;
-
- /// Compute hash for Int96 value by using its plain encoding result.
- ///
- /// @param value the value to hash.
- /// @return hash result.
- virtual uint64_t Hash(const Int96* value) const = 0;
-
- /// Compute hash for ByteArray value by using its plain encoding result.
- ///
- /// @param value the value to hash.
- /// @return hash result.
- virtual uint64_t Hash(const ByteArray* value) const = 0;
-
- /// Compute hash for fixed byte array value by using its plain encoding result.
- ///
- /// @param value the value to hash.
- /// @return hash result.
- virtual uint64_t Hash(const FLBA* value, uint32_t len) const = 0;
-
- virtual ~Hasher() = default;
-};
-
-} // namespace parquet
-
-#endif // PARQUET_HASHER_H
diff --git a/src/parquet/metadata-test.cc b/src/parquet/metadata-test.cc
deleted file mode 100644
index 53653bd..0000000
--- a/src/parquet/metadata-test.cc
+++ /dev/null
@@ -1,231 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#include "parquet/metadata.h"
-#include <gtest/gtest.h>
-#include "parquet/schema.h"
-#include "parquet/statistics.h"
-#include "parquet/types.h"
-
-namespace parquet {
-
-namespace metadata {
-
-TEST(Metadata, TestBuildAccess) {
- parquet::schema::NodeVector fields;
- parquet::schema::NodePtr root;
- parquet::SchemaDescriptor schema;
-
- WriterProperties::Builder prop_builder;
-
- std::shared_ptr<WriterProperties> props =
- prop_builder.version(ParquetVersion::PARQUET_2_0)->build();
-
- fields.push_back(parquet::schema::Int32("int_col", Repetition::REQUIRED));
- fields.push_back(parquet::schema::Float("float_col", Repetition::REQUIRED));
- root = parquet::schema::GroupNode::Make("schema", Repetition::REPEATED, fields);
- schema.Init(root);
-
- int64_t nrows = 1000;
- int32_t int_min = 100, int_max = 200;
- EncodedStatistics stats_int;
- stats_int.set_null_count(0)
- .set_distinct_count(nrows)
- .set_min(std::string(reinterpret_cast<const char*>(&int_min), 4))
- .set_max(std::string(reinterpret_cast<const char*>(&int_max), 4));
- EncodedStatistics stats_float;
- float float_min = 100.100f, float_max = 200.200f;
- stats_float.set_null_count(0)
- .set_distinct_count(nrows)
- .set_min(std::string(reinterpret_cast<const char*>(&float_min), 4))
- .set_max(std::string(reinterpret_cast<const char*>(&float_max), 4));
-
- auto f_builder = FileMetaDataBuilder::Make(&schema, props);
- auto rg1_builder = f_builder->AppendRowGroup();
- auto rg2_builder = f_builder->AppendRowGroup();
-
- // Write the metadata
- // rowgroup1 metadata
- auto col1_builder = rg1_builder->NextColumnChunk();
- auto col2_builder = rg1_builder->NextColumnChunk();
- // column metadata
- col1_builder->SetStatistics(true, stats_int);
- col2_builder->SetStatistics(true, stats_float);
- col1_builder->Finish(nrows / 2, 4, 0, 10, 512, 600, true, false);
- col2_builder->Finish(nrows / 2, 24, 0, 30, 512, 600, true, false);
-
- rg1_builder->set_num_rows(nrows / 2);
- rg1_builder->Finish(1024);
-
- // rowgroup2 metadata
- col1_builder = rg2_builder->NextColumnChunk();
- col2_builder = rg2_builder->NextColumnChunk();
- // column metadata
- col1_builder->SetStatistics(true, stats_int);
- col2_builder->SetStatistics(true, stats_float);
- col1_builder->Finish(nrows / 2, 6, 0, 10, 512, 600, true, false);
- col2_builder->Finish(nrows / 2, 16, 0, 26, 512, 600, true, false);
-
- rg2_builder->set_num_rows(nrows / 2);
- rg2_builder->Finish(1024);
-
- // Read the metadata
- auto f_accessor = f_builder->Finish();
-
- // file metadata
- ASSERT_EQ(nrows, f_accessor->num_rows());
- ASSERT_LE(0, static_cast<int>(f_accessor->size()));
- ASSERT_EQ(2, f_accessor->num_row_groups());
- ASSERT_EQ(ParquetVersion::PARQUET_2_0, f_accessor->version());
- ASSERT_EQ(DEFAULT_CREATED_BY, f_accessor->created_by());
- ASSERT_EQ(3, f_accessor->num_schema_elements());
-
- // row group1 metadata
- auto rg1_accessor = f_accessor->RowGroup(0);
- ASSERT_EQ(2, rg1_accessor->num_columns());
- ASSERT_EQ(nrows / 2, rg1_accessor->num_rows());
- ASSERT_EQ(1024, rg1_accessor->total_byte_size());
-
- auto rg1_column1 = rg1_accessor->ColumnChunk(0);
- auto rg1_column2 = rg1_accessor->ColumnChunk(1);
- ASSERT_EQ(true, rg1_column1->is_stats_set());
- ASSERT_EQ(true, rg1_column2->is_stats_set());
- ASSERT_EQ(stats_float.min(), rg1_column2->statistics()->EncodeMin());
- ASSERT_EQ(stats_float.max(), rg1_column2->statistics()->EncodeMax());
- ASSERT_EQ(stats_int.min(), rg1_column1->statistics()->EncodeMin());
- ASSERT_EQ(stats_int.max(), rg1_column1->statistics()->EncodeMax());
- ASSERT_EQ(0, rg1_column1->statistics()->null_count());
- ASSERT_EQ(0, rg1_column2->statistics()->null_count());
- ASSERT_EQ(nrows, rg1_column1->statistics()->distinct_count());
- ASSERT_EQ(nrows, rg1_column2->statistics()->distinct_count());
- ASSERT_EQ(DEFAULT_COMPRESSION_TYPE, rg1_column1->compression());
- ASSERT_EQ(DEFAULT_COMPRESSION_TYPE, rg1_column2->compression());
- ASSERT_EQ(nrows / 2, rg1_column1->num_values());
- ASSERT_EQ(nrows / 2, rg1_column2->num_values());
- ASSERT_EQ(3, rg1_column1->encodings().size());
- ASSERT_EQ(3, rg1_column2->encodings().size());
- ASSERT_EQ(512, rg1_column1->total_compressed_size());
- ASSERT_EQ(512, rg1_column2->total_compressed_size());
- ASSERT_EQ(600, rg1_column1->total_uncompressed_size());
- ASSERT_EQ(600, rg1_column2->total_uncompressed_size());
- ASSERT_EQ(4, rg1_column1->dictionary_page_offset());
- ASSERT_EQ(24, rg1_column2->dictionary_page_offset());
- ASSERT_EQ(10, rg1_column1->data_page_offset());
- ASSERT_EQ(30, rg1_column2->data_page_offset());
-
- auto rg2_accessor = f_accessor->RowGroup(1);
- ASSERT_EQ(2, rg2_accessor->num_columns());
- ASSERT_EQ(nrows / 2, rg2_accessor->num_rows());
- ASSERT_EQ(1024, rg2_accessor->total_byte_size());
-
- auto rg2_column1 = rg2_accessor->ColumnChunk(0);
- auto rg2_column2 = rg2_accessor->ColumnChunk(1);
- ASSERT_EQ(true, rg2_column1->is_stats_set());
- ASSERT_EQ(true, rg2_column2->is_stats_set());
- ASSERT_EQ(stats_float.min(), rg2_column2->statistics()->EncodeMin());
- ASSERT_EQ(stats_float.max(), rg2_column2->statistics()->EncodeMax());
- ASSERT_EQ(stats_int.min(), rg1_column1->statistics()->EncodeMin());
- ASSERT_EQ(stats_int.max(), rg1_column1->statistics()->EncodeMax());
- ASSERT_EQ(0, rg2_column1->statistics()->null_count());
- ASSERT_EQ(0, rg2_column2->statistics()->null_count());
- ASSERT_EQ(nrows, rg2_column1->statistics()->distinct_count());
- ASSERT_EQ(nrows, rg2_column2->statistics()->distinct_count());
- ASSERT_EQ(nrows / 2, rg2_column1->num_values());
- ASSERT_EQ(nrows / 2, rg2_column2->num_values());
- ASSERT_EQ(DEFAULT_COMPRESSION_TYPE, rg2_column1->compression());
- ASSERT_EQ(DEFAULT_COMPRESSION_TYPE, rg2_column2->compression());
- ASSERT_EQ(3, rg2_column1->encodings().size());
- ASSERT_EQ(3, rg2_column2->encodings().size());
- ASSERT_EQ(512, rg2_column1->total_compressed_size());
- ASSERT_EQ(512, rg2_column2->total_compressed_size());
- ASSERT_EQ(600, rg2_column1->total_uncompressed_size());
- ASSERT_EQ(600, rg2_column2->total_uncompressed_size());
- ASSERT_EQ(6, rg2_column1->dictionary_page_offset());
- ASSERT_EQ(16, rg2_column2->dictionary_page_offset());
- ASSERT_EQ(10, rg2_column1->data_page_offset());
- ASSERT_EQ(26, rg2_column2->data_page_offset());
-}
-
-TEST(Metadata, TestV1Version) {
- // PARQUET-839
- parquet::schema::NodeVector fields;
- parquet::schema::NodePtr root;
- parquet::SchemaDescriptor schema;
-
- WriterProperties::Builder prop_builder;
-
- std::shared_ptr<WriterProperties> props =
- prop_builder.version(ParquetVersion::PARQUET_1_0)->build();
-
- fields.push_back(parquet::schema::Int32("int_col", Repetition::REQUIRED));
- fields.push_back(parquet::schema::Float("float_col", Repetition::REQUIRED));
- root = parquet::schema::GroupNode::Make("schema", Repetition::REPEATED, fields);
- schema.Init(root);
-
- auto f_builder = FileMetaDataBuilder::Make(&schema, props);
-
- // Read the metadata
- auto f_accessor = f_builder->Finish();
-
- // file metadata
- ASSERT_EQ(ParquetVersion::PARQUET_1_0, f_accessor->version());
-}
-
-TEST(ApplicationVersion, Basics) {
- ApplicationVersion version("parquet-mr version 1.7.9");
- ApplicationVersion version1("parquet-mr version 1.8.0");
- ApplicationVersion version2("parquet-cpp version 1.0.0");
- ApplicationVersion version3("");
- ApplicationVersion version4("parquet-mr version 1.5.0ab-cdh5.5.0+cd (build abcd)");
- ApplicationVersion version5("parquet-mr");
-
- ASSERT_EQ("parquet-mr", version.application_);
- ASSERT_EQ(1, version.version.major);
- ASSERT_EQ(7, version.version.minor);
- ASSERT_EQ(9, version.version.patch);
-
- ASSERT_EQ("parquet-cpp", version2.application_);
- ASSERT_EQ(1, version2.version.major);
- ASSERT_EQ(0, version2.version.minor);
- ASSERT_EQ(0, version2.version.patch);
-
- ASSERT_EQ("parquet-mr", version4.application_);
- ASSERT_EQ("abcd", version4.build_);
- ASSERT_EQ(1, version4.version.major);
- ASSERT_EQ(5, version4.version.minor);
- ASSERT_EQ(0, version4.version.patch);
- ASSERT_EQ("ab", version4.version.unknown);
- ASSERT_EQ("cdh5.5.0", version4.version.pre_release);
- ASSERT_EQ("cd", version4.version.build_info);
-
- ASSERT_EQ("parquet-mr", version5.application_);
- ASSERT_EQ(0, version5.version.major);
- ASSERT_EQ(0, version5.version.minor);
- ASSERT_EQ(0, version5.version.patch);
-
- ASSERT_EQ(true, version.VersionLt(version1));
-
- ASSERT_FALSE(version1.HasCorrectStatistics(Type::INT96, SortOrder::UNKNOWN));
- ASSERT_TRUE(version.HasCorrectStatistics(Type::INT32, SortOrder::SIGNED));
- ASSERT_FALSE(version.HasCorrectStatistics(Type::BYTE_ARRAY, SortOrder::SIGNED));
- ASSERT_TRUE(version1.HasCorrectStatistics(Type::BYTE_ARRAY, SortOrder::SIGNED));
- ASSERT_TRUE(
- version3.HasCorrectStatistics(Type::FIXED_LEN_BYTE_ARRAY, SortOrder::SIGNED));
-}
-
-} // namespace metadata
-} // namespace parquet
diff --git a/src/parquet/metadata.cc b/src/parquet/metadata.cc
deleted file mode 100644
index 1cab51f..0000000
--- a/src/parquet/metadata.cc
+++ /dev/null
@@ -1,910 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#include <algorithm>
-#include <string>
-#include <vector>
-
-#include "parquet/exception.h"
-#include "parquet/metadata.h"
-#include "parquet/schema-internal.h"
-#include "parquet/schema.h"
-#include "parquet/thrift.h"
-#include "parquet/util/memory.h"
-
-#include <boost/algorithm/string.hpp>
-#include <boost/regex.hpp>
-
-namespace parquet {
-
-const ApplicationVersion& ApplicationVersion::PARQUET_251_FIXED_VERSION() {
- static ApplicationVersion version("parquet-mr", 1, 8, 0);
- return version;
-}
-
-const ApplicationVersion& ApplicationVersion::PARQUET_816_FIXED_VERSION() {
- static ApplicationVersion version("parquet-mr", 1, 2, 9);
- return version;
-}
-
-const ApplicationVersion& ApplicationVersion::PARQUET_CPP_FIXED_STATS_VERSION() {
- static ApplicationVersion version("parquet-cpp", 1, 3, 0);
- return version;
-}
-
-template <typename DType>
-static std::shared_ptr<RowGroupStatistics> MakeTypedColumnStats(
- const format::ColumnMetaData& metadata, const ColumnDescriptor* descr) {
- // If ColumnOrder is defined, return max_value and min_value
- if (descr->column_order().get_order() == ColumnOrder::TYPE_DEFINED_ORDER) {
- return std::make_shared<TypedRowGroupStatistics<DType>>(
- descr, metadata.statistics.min_value, metadata.statistics.max_value,
- metadata.num_values - metadata.statistics.null_count,
- metadata.statistics.null_count, metadata.statistics.distinct_count, true);
- }
- // Default behavior
- return std::make_shared<TypedRowGroupStatistics<DType>>(
- descr, metadata.statistics.min, metadata.statistics.max,
- metadata.num_values - metadata.statistics.null_count,
- metadata.statistics.null_count, metadata.statistics.distinct_count,
- metadata.statistics.__isset.max || metadata.statistics.__isset.min);
-}
-
-std::shared_ptr<RowGroupStatistics> MakeColumnStats(
- const format::ColumnMetaData& meta_data, const ColumnDescriptor* descr) {
- switch (static_cast<Type::type>(meta_data.type)) {
- case Type::BOOLEAN:
- return MakeTypedColumnStats<BooleanType>(meta_data, descr);
- case Type::INT32:
- return MakeTypedColumnStats<Int32Type>(meta_data, descr);
- case Type::INT64:
- return MakeTypedColumnStats<Int64Type>(meta_data, descr);
- case Type::INT96:
- return MakeTypedColumnStats<Int96Type>(meta_data, descr);
- case Type::DOUBLE:
- return MakeTypedColumnStats<DoubleType>(meta_data, descr);
- case Type::FLOAT:
- return MakeTypedColumnStats<FloatType>(meta_data, descr);
- case Type::BYTE_ARRAY:
- return MakeTypedColumnStats<ByteArrayType>(meta_data, descr);
- case Type::FIXED_LEN_BYTE_ARRAY:
- return MakeTypedColumnStats<FLBAType>(meta_data, descr);
- }
- throw ParquetException("Can't decode page statistics for selected column type");
-}
-
-// MetaData Accessor
-// ColumnChunk metadata
-class ColumnChunkMetaData::ColumnChunkMetaDataImpl {
- public:
- explicit ColumnChunkMetaDataImpl(const format::ColumnChunk* column,
- const ColumnDescriptor* descr,
- const ApplicationVersion* writer_version)
- : column_(column), descr_(descr), writer_version_(writer_version) {
- const format::ColumnMetaData& meta_data = column->meta_data;
- for (auto encoding : meta_data.encodings) {
- encodings_.push_back(FromThrift(encoding));
- }
- stats_ = nullptr;
- }
- ~ColumnChunkMetaDataImpl() {}
-
- // column chunk
- inline int64_t file_offset() const { return column_->file_offset; }
- inline const std::string& file_path() const { return column_->file_path; }
-
- // column metadata
- inline Type::type type() const { return FromThrift(column_->meta_data.type); }
-
- inline int64_t num_values() const { return column_->meta_data.num_values; }
-
- std::shared_ptr<schema::ColumnPath> path_in_schema() {
- return std::make_shared<schema::ColumnPath>(column_->meta_data.path_in_schema);
- }
-
- // Check if statistics are set and are valid
- // 1) Must be set in the metadata
- // 2) Statistics must not be corrupted
- // 3) parquet-mr and parquet-cpp write statistics by SIGNED order comparison.
- // The statistics are corrupted if the type requires UNSIGNED order comparison.
- // Eg: UTF8
- inline bool is_stats_set() const {
- DCHECK(writer_version_ != nullptr);
- return column_->meta_data.__isset.statistics &&
- writer_version_->HasCorrectStatistics(type(), descr_->sort_order());
- }
-
- inline std::shared_ptr<RowGroupStatistics> statistics() const {
- if (stats_ == nullptr && is_stats_set()) {
- stats_ = MakeColumnStats(column_->meta_data, descr_);
- }
- return stats_;
- }
-
- inline Compression::type compression() const {
- return FromThrift(column_->meta_data.codec);
- }
-
- const std::vector<Encoding::type>& encodings() const { return encodings_; }
-
- inline bool has_dictionary_page() const {
- return column_->meta_data.__isset.dictionary_page_offset;
- }
-
- inline int64_t dictionary_page_offset() const {
- return column_->meta_data.dictionary_page_offset;
- }
-
- inline int64_t data_page_offset() const { return column_->meta_data.data_page_offset; }
-
- inline bool has_index_page() const {
- return column_->meta_data.__isset.index_page_offset;
- }
-
- inline int64_t index_page_offset() const {
- return column_->meta_data.index_page_offset;
- }
-
- inline int64_t total_compressed_size() const {
- return column_->meta_data.total_compressed_size;
- }
-
- inline int64_t total_uncompressed_size() const {
- return column_->meta_data.total_uncompressed_size;
- }
-
- private:
- mutable std::shared_ptr<RowGroupStatistics> stats_;
- std::vector<Encoding::type> encodings_;
- const format::ColumnChunk* column_;
- const ColumnDescriptor* descr_;
- const ApplicationVersion* writer_version_;
-};
-
-std::unique_ptr<ColumnChunkMetaData> ColumnChunkMetaData::Make(
- const uint8_t* metadata, const ColumnDescriptor* descr,
- const ApplicationVersion* writer_version) {
- return std::unique_ptr<ColumnChunkMetaData>(
- new ColumnChunkMetaData(metadata, descr, writer_version));
-}
-
-ColumnChunkMetaData::ColumnChunkMetaData(const uint8_t* metadata,
- const ColumnDescriptor* descr,
- const ApplicationVersion* writer_version)
- : impl_{std::unique_ptr<ColumnChunkMetaDataImpl>(new ColumnChunkMetaDataImpl(
- reinterpret_cast<const format::ColumnChunk*>(metadata), descr,
- writer_version))} {}
-ColumnChunkMetaData::~ColumnChunkMetaData() {}
-
-// column chunk
-int64_t ColumnChunkMetaData::file_offset() const { return impl_->file_offset(); }
-
-const std::string& ColumnChunkMetaData::file_path() const { return impl_->file_path(); }
-
-// column metadata
-Type::type ColumnChunkMetaData::type() const { return impl_->type(); }
-
-int64_t ColumnChunkMetaData::num_values() const { return impl_->num_values(); }
-
-std::shared_ptr<schema::ColumnPath> ColumnChunkMetaData::path_in_schema() const {
- return impl_->path_in_schema();
-}
-
-std::shared_ptr<RowGroupStatistics> ColumnChunkMetaData::statistics() const {
- return impl_->statistics();
-}
-
-bool ColumnChunkMetaData::is_stats_set() const { return impl_->is_stats_set(); }
-
-bool ColumnChunkMetaData::has_dictionary_page() const {
- return impl_->has_dictionary_page();
-}
-
-int64_t ColumnChunkMetaData::dictionary_page_offset() const {
- return impl_->dictionary_page_offset();
-}
-
-int64_t ColumnChunkMetaData::data_page_offset() const {
- return impl_->data_page_offset();
-}
-
-bool ColumnChunkMetaData::has_index_page() const { return impl_->has_index_page(); }
-
-int64_t ColumnChunkMetaData::index_page_offset() const {
- return impl_->index_page_offset();
-}
-
-Compression::type ColumnChunkMetaData::compression() const {
- return impl_->compression();
-}
-
-const std::vector<Encoding::type>& ColumnChunkMetaData::encodings() const {
- return impl_->encodings();
-}
-
-int64_t ColumnChunkMetaData::total_uncompressed_size() const {
- return impl_->total_uncompressed_size();
-}
-
-int64_t ColumnChunkMetaData::total_compressed_size() const {
- return impl_->total_compressed_size();
-}
-
-// row-group metadata
-class RowGroupMetaData::RowGroupMetaDataImpl {
- public:
- explicit RowGroupMetaDataImpl(const format::RowGroup* row_group,
- const SchemaDescriptor* schema,
- const ApplicationVersion* writer_version)
- : row_group_(row_group), schema_(schema), writer_version_(writer_version) {}
- ~RowGroupMetaDataImpl() {}
-
- inline int num_columns() const { return static_cast<int>(row_group_->columns.size()); }
-
- inline int64_t num_rows() const { return row_group_->num_rows; }
-
- inline int64_t total_byte_size() const { return row_group_->total_byte_size; }
-
- inline const SchemaDescriptor* schema() const { return schema_; }
-
- std::unique_ptr<ColumnChunkMetaData> ColumnChunk(int i) {
- if (!(i < num_columns())) {
- std::stringstream ss;
- ss << "The file only has " << num_columns()
- << " columns, requested metadata for column: " << i;
- throw ParquetException(ss.str());
- }
- return ColumnChunkMetaData::Make(
- reinterpret_cast<const uint8_t*>(&row_group_->columns[i]), schema_->Column(i),
- writer_version_);
- }
-
- private:
- const format::RowGroup* row_group_;
- const SchemaDescriptor* schema_;
- const ApplicationVersion* writer_version_;
-};
-
-std::unique_ptr<RowGroupMetaData> RowGroupMetaData::Make(
- const uint8_t* metadata, const SchemaDescriptor* schema,
- const ApplicationVersion* writer_version) {
- return std::unique_ptr<RowGroupMetaData>(
- new RowGroupMetaData(metadata, schema, writer_version));
-}
-
-RowGroupMetaData::RowGroupMetaData(const uint8_t* metadata,
- const SchemaDescriptor* schema,
- const ApplicationVersion* writer_version)
- : impl_{std::unique_ptr<RowGroupMetaDataImpl>(new RowGroupMetaDataImpl(
- reinterpret_cast<const format::RowGroup*>(metadata), schema, writer_version))} {
-}
-RowGroupMetaData::~RowGroupMetaData() {}
-
-int RowGroupMetaData::num_columns() const { return impl_->num_columns(); }
-
-int64_t RowGroupMetaData::num_rows() const { return impl_->num_rows(); }
-
-int64_t RowGroupMetaData::total_byte_size() const { return impl_->total_byte_size(); }
-
-const SchemaDescriptor* RowGroupMetaData::schema() const { return impl_->schema(); }
-
-std::unique_ptr<ColumnChunkMetaData> RowGroupMetaData::ColumnChunk(int i) const {
- return impl_->ColumnChunk(i);
-}
-
-// file metadata
-class FileMetaData::FileMetaDataImpl {
- public:
- FileMetaDataImpl() : metadata_len_(0) {}
-
- explicit FileMetaDataImpl(const uint8_t* metadata, uint32_t* metadata_len)
- : metadata_len_(0) {
- metadata_.reset(new format::FileMetaData);
- DeserializeThriftMsg(metadata, metadata_len, metadata_.get());
- metadata_len_ = *metadata_len;
-
- if (metadata_->__isset.created_by) {
- writer_version_ = ApplicationVersion(metadata_->created_by);
- } else {
- writer_version_ = ApplicationVersion("unknown 0.0.0");
- }
-
- InitSchema();
- InitColumnOrders();
- InitKeyValueMetadata();
- }
- ~FileMetaDataImpl() {}
-
- inline uint32_t size() const { return metadata_len_; }
- inline int num_columns() const { return schema_.num_columns(); }
- inline int64_t num_rows() const { return metadata_->num_rows; }
- inline int num_row_groups() const {
- return static_cast<int>(metadata_->row_groups.size());
- }
- inline int32_t version() const { return metadata_->version; }
- inline const std::string& created_by() const { return metadata_->created_by; }
- inline int num_schema_elements() const {
- return static_cast<int>(metadata_->schema.size());
- }
-
- const ApplicationVersion& writer_version() const { return writer_version_; }
-
- void WriteTo(OutputStream* dst) const {
- SerializeThriftMsg(metadata_.get(), 1024, dst);
- }
-
- std::unique_ptr<RowGroupMetaData> RowGroup(int i) {
- if (!(i < num_row_groups())) {
- std::stringstream ss;
- ss << "The file only has " << num_row_groups()
- << " row groups, requested metadata for row group: " << i;
- throw ParquetException(ss.str());
- }
- return RowGroupMetaData::Make(
- reinterpret_cast<const uint8_t*>(&metadata_->row_groups[i]), &schema_,
- &writer_version_);
- }
-
- const SchemaDescriptor* schema() const { return &schema_; }
-
- std::shared_ptr<const KeyValueMetadata> key_value_metadata() const {
- return key_value_metadata_;
- }
-
- private:
- friend FileMetaDataBuilder;
- uint32_t metadata_len_;
- std::unique_ptr<format::FileMetaData> metadata_;
- void InitSchema() {
- schema::FlatSchemaConverter converter(&metadata_->schema[0],
- static_cast<int>(metadata_->schema.size()));
- schema_.Init(converter.Convert());
- }
- void InitColumnOrders() {
- // update ColumnOrder
- std::vector<parquet::ColumnOrder> column_orders;
- if (metadata_->__isset.column_orders) {
- for (auto column_order : metadata_->column_orders) {
- if (column_order.__isset.TYPE_ORDER) {
- column_orders.push_back(ColumnOrder::type_defined_);
- } else {
- column_orders.push_back(ColumnOrder::undefined_);
- }
- }
- } else {
- column_orders.resize(schema_.num_columns(), ColumnOrder::undefined_);
- }
-
- schema_.updateColumnOrders(column_orders);
- }
- SchemaDescriptor schema_;
- ApplicationVersion writer_version_;
-
- void InitKeyValueMetadata() {
- std::shared_ptr<KeyValueMetadata> metadata = nullptr;
- if (metadata_->__isset.key_value_metadata) {
- metadata = std::make_shared<KeyValueMetadata>();
- for (const auto& it : metadata_->key_value_metadata) {
- metadata->Append(it.key, it.value);
- }
- }
- key_value_metadata_ = metadata;
- }
-
- std::shared_ptr<const KeyValueMetadata> key_value_metadata_;
-};
-
-std::shared_ptr<FileMetaData> FileMetaData::Make(const uint8_t* metadata,
- uint32_t* metadata_len) {
- // This FileMetaData ctor is private, not compatible with std::make_shared
- return std::shared_ptr<FileMetaData>(new FileMetaData(metadata, metadata_len));
-}
-
-FileMetaData::FileMetaData(const uint8_t* metadata, uint32_t* metadata_len)
- : impl_{std::unique_ptr<FileMetaDataImpl>(
- new FileMetaDataImpl(metadata, metadata_len))} {}
-
-FileMetaData::FileMetaData()
- : impl_{std::unique_ptr<FileMetaDataImpl>(new FileMetaDataImpl())} {}
-
-FileMetaData::~FileMetaData() {}
-
-std::unique_ptr<RowGroupMetaData> FileMetaData::RowGroup(int i) const {
- return impl_->RowGroup(i);
-}
-
-uint32_t FileMetaData::size() const { return impl_->size(); }
-
-int FileMetaData::num_columns() const { return impl_->num_columns(); }
-
-int64_t FileMetaData::num_rows() const { return impl_->num_rows(); }
-
-int FileMetaData::num_row_groups() const { return impl_->num_row_groups(); }
-
-ParquetVersion::type FileMetaData::version() const {
- switch (impl_->version()) {
- case 1:
- return ParquetVersion::PARQUET_1_0;
- case 2:
- return ParquetVersion::PARQUET_2_0;
- default:
- // Improperly set version, assuming Parquet 1.0
- break;
- }
- return ParquetVersion::PARQUET_1_0;
-}
-
-const ApplicationVersion& FileMetaData::writer_version() const {
- return impl_->writer_version();
-}
-
-const std::string& FileMetaData::created_by() const { return impl_->created_by(); }
-
-int FileMetaData::num_schema_elements() const { return impl_->num_schema_elements(); }
-
-const SchemaDescriptor* FileMetaData::schema() const { return impl_->schema(); }
-
-std::shared_ptr<const KeyValueMetadata> FileMetaData::key_value_metadata() const {
- return impl_->key_value_metadata();
-}
-
-void FileMetaData::WriteTo(OutputStream* dst) const { return impl_->WriteTo(dst); }
-
-ApplicationVersion::ApplicationVersion(const std::string& application, int major,
- int minor, int patch)
- : application_(application), version{major, minor, patch, "", "", ""} {}
-
-ApplicationVersion::ApplicationVersion(const std::string& created_by) {
- boost::regex app_regex{ApplicationVersion::APPLICATION_FORMAT};
- boost::regex ver_regex{ApplicationVersion::VERSION_FORMAT};
- boost::smatch app_matches;
- boost::smatch ver_matches;
-
- std::string created_by_lower = created_by;
- std::transform(created_by_lower.begin(), created_by_lower.end(),
- created_by_lower.begin(), ::tolower);
-
- bool app_success = boost::regex_match(created_by_lower, app_matches, app_regex);
- bool ver_success = false;
- std::string version_str;
-
- if (app_success && app_matches.size() >= 4) {
- // first match is the entire string. sub-matches start from second.
- application_ = app_matches[1];
- version_str = app_matches[3];
- build_ = app_matches[4];
- ver_success = boost::regex_match(version_str, ver_matches, ver_regex);
- } else {
- application_ = "unknown";
- }
-
- if (ver_success && ver_matches.size() >= 7) {
- version.major = atoi(ver_matches[1].str().c_str());
- version.minor = atoi(ver_matches[2].str().c_str());
- version.patch = atoi(ver_matches[3].str().c_str());
- version.unknown = ver_matches[4].str();
- version.pre_release = ver_matches[5].str();
- version.build_info = ver_matches[6].str();
- } else {
- version.major = 0;
- version.minor = 0;
- version.patch = 0;
- }
-}
-
-bool ApplicationVersion::VersionLt(const ApplicationVersion& other_version) const {
- if (application_ != other_version.application_) return false;
-
- if (version.major < other_version.version.major) return true;
- if (version.major > other_version.version.major) return false;
- DCHECK_EQ(version.major, other_version.version.major);
- if (version.minor < other_version.version.minor) return true;
- if (version.minor > other_version.version.minor) return false;
- DCHECK_EQ(version.minor, other_version.version.minor);
- return version.patch < other_version.version.patch;
-}
-
-bool ApplicationVersion::VersionEq(const ApplicationVersion& other_version) const {
- return application_ == other_version.application_ &&
- version.major == other_version.version.major &&
- version.minor == other_version.version.minor &&
- version.patch == other_version.version.patch;
-}
-
-// Reference:
-// parquet-mr/parquet-column/src/main/java/org/apache/parquet/CorruptStatistics.java
-// PARQUET-686 has more disussion on statistics
-bool ApplicationVersion::HasCorrectStatistics(Type::type col_type,
- SortOrder::type sort_order) const {
- // Parquet cpp version 1.3.0 onwards stats are computed correctly for all types
- if ((application_ != "parquet-cpp") || (VersionLt(PARQUET_CPP_FIXED_STATS_VERSION()))) {
- // Only SIGNED are valid
- if (SortOrder::SIGNED != sort_order) {
- return false;
- }
-
- // Statistics of other types are OK
- if (col_type != Type::FIXED_LEN_BYTE_ARRAY && col_type != Type::BYTE_ARRAY) {
- return true;
- }
- }
- // created_by is not populated, which could have been caused by
- // parquet-mr during the same time as PARQUET-251, see PARQUET-297
- if (application_ == "unknown") {
- return true;
- }
-
- // Unknown sort order has incorrect stats
- if (SortOrder::UNKNOWN == sort_order) {
- return false;
- }
-
- // PARQUET-251
- if (VersionLt(PARQUET_251_FIXED_VERSION())) {
- return false;
- }
-
- return true;
-}
-
-// MetaData Builders
-// row-group metadata
-class ColumnChunkMetaDataBuilder::ColumnChunkMetaDataBuilderImpl {
- public:
- explicit ColumnChunkMetaDataBuilderImpl(const std::shared_ptr<WriterProperties>& props,
- const ColumnDescriptor* column,
- uint8_t* contents)
- : properties_(props), column_(column) {
- column_chunk_ = reinterpret_cast<format::ColumnChunk*>(contents);
- column_chunk_->meta_data.__set_type(ToThrift(column->physical_type()));
- column_chunk_->meta_data.__set_path_in_schema(column->path()->ToDotVector());
- column_chunk_->meta_data.__set_codec(
- ToThrift(properties_->compression(column->path())));
- }
- ~ColumnChunkMetaDataBuilderImpl() {}
-
- // column chunk
- void set_file_path(const std::string& val) { column_chunk_->__set_file_path(val); }
-
- // column metadata
- void SetStatistics(bool is_signed, const EncodedStatistics& val) {
- format::Statistics stats;
- stats.null_count = val.null_count;
- stats.distinct_count = val.distinct_count;
- stats.max_value = val.max();
- stats.min_value = val.min();
- stats.__isset.min_value = val.has_min;
- stats.__isset.max_value = val.has_max;
- stats.__isset.null_count = val.has_null_count;
- stats.__isset.distinct_count = val.has_distinct_count;
- // If the order is SIGNED, then the old min/max values must be set too.
- // This for backward compatibility
- if (is_signed) {
- stats.max = val.max();
- stats.min = val.min();
- stats.__isset.min = val.has_min;
- stats.__isset.max = val.has_max;
- }
-
- column_chunk_->meta_data.__set_statistics(stats);
- }
-
- void Finish(int64_t num_values, int64_t dictionary_page_offset,
- int64_t index_page_offset, int64_t data_page_offset,
- int64_t compressed_size, int64_t uncompressed_size, bool has_dictionary,
- bool dictionary_fallback) {
- if (dictionary_page_offset > 0) {
- column_chunk_->meta_data.__set_dictionary_page_offset(dictionary_page_offset);
- column_chunk_->__set_file_offset(dictionary_page_offset + compressed_size);
- } else {
- column_chunk_->__set_file_offset(data_page_offset + compressed_size);
- }
- column_chunk_->__isset.meta_data = true;
- column_chunk_->meta_data.__set_num_values(num_values);
- if (index_page_offset >= 0) {
- column_chunk_->meta_data.__set_index_page_offset(index_page_offset);
- }
- column_chunk_->meta_data.__set_data_page_offset(data_page_offset);
- column_chunk_->meta_data.__set_total_uncompressed_size(uncompressed_size);
- column_chunk_->meta_data.__set_total_compressed_size(compressed_size);
- std::vector<format::Encoding::type> thrift_encodings;
- if (has_dictionary) {
- thrift_encodings.push_back(ToThrift(properties_->dictionary_index_encoding()));
- if (properties_->version() == ParquetVersion::PARQUET_1_0) {
- thrift_encodings.push_back(ToThrift(Encoding::PLAIN));
- } else {
- thrift_encodings.push_back(ToThrift(properties_->dictionary_page_encoding()));
- }
- } else { // Dictionary not enabled
- thrift_encodings.push_back(ToThrift(properties_->encoding(column_->path())));
- }
- thrift_encodings.push_back(ToThrift(Encoding::RLE));
- // Only PLAIN encoding is supported for fallback in V1
- // TODO(majetideepak): Use user specified encoding for V2
- if (dictionary_fallback) {
- thrift_encodings.push_back(ToThrift(Encoding::PLAIN));
- }
- column_chunk_->meta_data.__set_encodings(thrift_encodings);
- }
-
- void WriteTo(OutputStream* sink) {
- SerializeThriftMsg(column_chunk_, sizeof(format::ColumnChunk), sink);
- }
-
- const ColumnDescriptor* descr() const { return column_; }
-
- private:
- format::ColumnChunk* column_chunk_;
- const std::shared_ptr<WriterProperties> properties_;
- const ColumnDescriptor* column_;
-};
-
-std::unique_ptr<ColumnChunkMetaDataBuilder> ColumnChunkMetaDataBuilder::Make(
- const std::shared_ptr<WriterProperties>& props, const ColumnDescriptor* column,
- uint8_t* contents) {
- return std::unique_ptr<ColumnChunkMetaDataBuilder>(
- new ColumnChunkMetaDataBuilder(props, column, contents));
-}
-
-ColumnChunkMetaDataBuilder::ColumnChunkMetaDataBuilder(
- const std::shared_ptr<WriterProperties>& props, const ColumnDescriptor* column,
- uint8_t* contents)
- : impl_{std::unique_ptr<ColumnChunkMetaDataBuilderImpl>(
- new ColumnChunkMetaDataBuilderImpl(props, column, contents))} {}
-
-ColumnChunkMetaDataBuilder::~ColumnChunkMetaDataBuilder() {}
-
-void ColumnChunkMetaDataBuilder::set_file_path(const std::string& path) {
- impl_->set_file_path(path);
-}
-
-void ColumnChunkMetaDataBuilder::Finish(int64_t num_values,
- int64_t dictionary_page_offset,
- int64_t index_page_offset,
- int64_t data_page_offset, int64_t compressed_size,
- int64_t uncompressed_size, bool has_dictionary,
- bool dictionary_fallback) {
- impl_->Finish(num_values, dictionary_page_offset, index_page_offset, data_page_offset,
- compressed_size, uncompressed_size, has_dictionary, dictionary_fallback);
-}
-
-void ColumnChunkMetaDataBuilder::WriteTo(OutputStream* sink) { impl_->WriteTo(sink); }
-
-const ColumnDescriptor* ColumnChunkMetaDataBuilder::descr() const {
- return impl_->descr();
-}
-
-void ColumnChunkMetaDataBuilder::SetStatistics(bool is_signed,
- const EncodedStatistics& result) {
- impl_->SetStatistics(is_signed, result);
-}
-
-class RowGroupMetaDataBuilder::RowGroupMetaDataBuilderImpl {
- public:
- explicit RowGroupMetaDataBuilderImpl(const std::shared_ptr<WriterProperties>& props,
- const SchemaDescriptor* schema, uint8_t* contents)
- : properties_(props), schema_(schema), current_column_(0) {
- row_group_ = reinterpret_cast<format::RowGroup*>(contents);
- InitializeColumns(schema->num_columns());
- }
- ~RowGroupMetaDataBuilderImpl() {}
-
- ColumnChunkMetaDataBuilder* NextColumnChunk() {
- if (!(current_column_ < num_columns())) {
- std::stringstream ss;
- ss << "The schema only has " << num_columns()
- << " columns, requested metadata for column: " << current_column_;
- throw ParquetException(ss.str());
- }
- auto column = schema_->Column(current_column_);
- auto column_builder = ColumnChunkMetaDataBuilder::Make(
- properties_, column,
- reinterpret_cast<uint8_t*>(&row_group_->columns[current_column_++]));
- auto column_builder_ptr = column_builder.get();
- column_builders_.push_back(std::move(column_builder));
- return column_builder_ptr;
- }
-
- int current_column() { return current_column_; }
-
- void Finish(int64_t total_bytes_written) {
- if (!(current_column_ == schema_->num_columns())) {
- std::stringstream ss;
- ss << "Only " << current_column_ - 1 << " out of " << schema_->num_columns()
- << " columns are initialized";
- throw ParquetException(ss.str());
- }
- int64_t total_byte_size = 0;
-
- for (int i = 0; i < schema_->num_columns(); i++) {
- if (!(row_group_->columns[i].file_offset >= 0)) {
- std::stringstream ss;
- ss << "Column " << i << " is not complete.";
- throw ParquetException(ss.str());
- }
- total_byte_size += row_group_->columns[i].meta_data.total_compressed_size;
- }
- DCHECK(total_bytes_written == total_byte_size)
- << "Total bytes in this RowGroup does not match with compressed sizes of columns";
-
- row_group_->__set_total_byte_size(total_byte_size);
- }
-
- void set_num_rows(int64_t num_rows) { row_group_->num_rows = num_rows; }
-
- int num_columns() { return static_cast<int>(row_group_->columns.size()); }
-
- int64_t num_rows() { return row_group_->num_rows; }
-
- private:
- void InitializeColumns(int ncols) { row_group_->columns.resize(ncols); }
-
- format::RowGroup* row_group_;
- const std::shared_ptr<WriterProperties> properties_;
- const SchemaDescriptor* schema_;
- std::vector<std::unique_ptr<ColumnChunkMetaDataBuilder>> column_builders_;
- int current_column_;
-};
-
-std::unique_ptr<RowGroupMetaDataBuilder> RowGroupMetaDataBuilder::Make(
- const std::shared_ptr<WriterProperties>& props, const SchemaDescriptor* schema_,
- uint8_t* contents) {
- return std::unique_ptr<RowGroupMetaDataBuilder>(
- new RowGroupMetaDataBuilder(props, schema_, contents));
-}
-
-RowGroupMetaDataBuilder::RowGroupMetaDataBuilder(
- const std::shared_ptr<WriterProperties>& props, const SchemaDescriptor* schema_,
- uint8_t* contents)
- : impl_{std::unique_ptr<RowGroupMetaDataBuilderImpl>(
- new RowGroupMetaDataBuilderImpl(props, schema_, contents))} {}
-
-RowGroupMetaDataBuilder::~RowGroupMetaDataBuilder() {}
-
-ColumnChunkMetaDataBuilder* RowGroupMetaDataBuilder::NextColumnChunk() {
- return impl_->NextColumnChunk();
-}
-
-int RowGroupMetaDataBuilder::current_column() const { return impl_->current_column(); }
-
-int RowGroupMetaDataBuilder::num_columns() { return impl_->num_columns(); }
-
-int64_t RowGroupMetaDataBuilder::num_rows() { return impl_->num_rows(); }
-
-void RowGroupMetaDataBuilder::set_num_rows(int64_t num_rows) {
- impl_->set_num_rows(num_rows);
-}
-
-void RowGroupMetaDataBuilder::Finish(int64_t total_bytes_written) {
- impl_->Finish(total_bytes_written);
-}
-
-// file metadata
-// TODO(PARQUET-595) Support key_value_metadata
-class FileMetaDataBuilder::FileMetaDataBuilderImpl {
- public:
- explicit FileMetaDataBuilderImpl(
- const SchemaDescriptor* schema, const std::shared_ptr<WriterProperties>& props,
- const std::shared_ptr<const KeyValueMetadata>& key_value_metadata)
- : properties_(props), schema_(schema), key_value_metadata_(key_value_metadata) {
- metadata_.reset(new format::FileMetaData());
- }
- ~FileMetaDataBuilderImpl() {}
-
- RowGroupMetaDataBuilder* AppendRowGroup() {
- auto row_group = std::unique_ptr<format::RowGroup>(new format::RowGroup());
- auto row_group_builder = RowGroupMetaDataBuilder::Make(
- properties_, schema_, reinterpret_cast<uint8_t*>(row_group.get()));
- RowGroupMetaDataBuilder* row_group_ptr = row_group_builder.get();
- row_group_builders_.push_back(std::move(row_group_builder));
- row_groups_.push_back(std::move(row_group));
- return row_group_ptr;
- }
-
- std::unique_ptr<FileMetaData> Finish() {
- int64_t total_rows = 0;
- std::vector<format::RowGroup> row_groups;
- for (auto row_group = row_groups_.begin(); row_group != row_groups_.end();
- row_group++) {
- auto rowgroup = *((*row_group).get());
- row_groups.push_back(rowgroup);
- total_rows += rowgroup.num_rows;
- }
- metadata_->__set_num_rows(total_rows);
- metadata_->__set_row_groups(row_groups);
-
- if (key_value_metadata_) {
- metadata_->key_value_metadata.clear();
- metadata_->key_value_metadata.reserve(key_value_metadata_->size());
- for (int64_t i = 0; i < key_value_metadata_->size(); ++i) {
- format::KeyValue kv_pair;
- kv_pair.__set_key(key_value_metadata_->key(i));
- kv_pair.__set_value(key_value_metadata_->value(i));
- metadata_->key_value_metadata.push_back(kv_pair);
- }
- metadata_->__isset.key_value_metadata = true;
- }
-
- int32_t file_version = 0;
- switch (properties_->version()) {
- case ParquetVersion::PARQUET_1_0:
- file_version = 1;
- break;
- case ParquetVersion::PARQUET_2_0:
- file_version = 2;
- break;
- default:
- break;
- }
- metadata_->__set_version(file_version);
- metadata_->__set_created_by(properties_->created_by());
-
- // Users cannot set the `ColumnOrder` since we donot not have user defined sort order
- // in the spec yet.
- // We always default to `TYPE_DEFINED_ORDER`. We can expose it in
- // the API once we have user defined sort orders in the Parquet format.
- // TypeDefinedOrder implies choose SortOrder based on LogicalType/PhysicalType
- format::TypeDefinedOrder type_defined_order;
- format::ColumnOrder column_order;
- column_order.__set_TYPE_ORDER(type_defined_order);
- column_order.__isset.TYPE_ORDER = true;
- metadata_->column_orders.resize(schema_->num_columns(), column_order);
- metadata_->__isset.column_orders = true;
-
- parquet::schema::SchemaFlattener flattener(
- static_cast<parquet::schema::GroupNode*>(schema_->schema_root().get()),
- &metadata_->schema);
- flattener.Flatten();
- auto file_meta_data = std::unique_ptr<FileMetaData>(new FileMetaData());
- file_meta_data->impl_->metadata_ = std::move(metadata_);
- file_meta_data->impl_->InitSchema();
- return file_meta_data;
- }
-
- protected:
- std::unique_ptr<format::FileMetaData> metadata_;
-
- private:
- const std::shared_ptr<WriterProperties> properties_;
- std::vector<std::unique_ptr<format::RowGroup>> row_groups_;
- std::vector<std::unique_ptr<RowGroupMetaDataBuilder>> row_group_builders_;
- const SchemaDescriptor* schema_;
- std::shared_ptr<const KeyValueMetadata> key_value_metadata_;
-};
-
-std::unique_ptr<FileMetaDataBuilder> FileMetaDataBuilder::Make(
- const SchemaDescriptor* schema, const std::shared_ptr<WriterProperties>& props,
- const std::shared_ptr<const KeyValueMetadata>& key_value_metadata) {
- return std::unique_ptr<FileMetaDataBuilder>(
- new FileMetaDataBuilder(schema, props, key_value_metadata));
-}
-
-FileMetaDataBuilder::FileMetaDataBuilder(
- const SchemaDescriptor* schema, const std::shared_ptr<WriterProperties>& props,
- const std::shared_ptr<const KeyValueMetadata>& key_value_metadata)
- : impl_{std::unique_ptr<FileMetaDataBuilderImpl>(
- new FileMetaDataBuilderImpl(schema, props, key_value_metadata))} {}
-
-FileMetaDataBuilder::~FileMetaDataBuilder() {}
-
-RowGroupMetaDataBuilder* FileMetaDataBuilder::AppendRowGroup() {
- return impl_->AppendRowGroup();
-}
-
-std::unique_ptr<FileMetaData> FileMetaDataBuilder::Finish() { return impl_->Finish(); }
-
-} // namespace parquet
diff --git a/src/parquet/metadata.h b/src/parquet/metadata.h
deleted file mode 100644
index 5d51e3d..0000000
--- a/src/parquet/metadata.h
+++ /dev/null
@@ -1,277 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#ifndef PARQUET_FILE_METADATA_H
-#define PARQUET_FILE_METADATA_H
-
-#include <set>
-#include <string>
-#include <vector>
-
-#include "arrow/util/key_value_metadata.h"
-
-#include "parquet/properties.h"
-#include "parquet/schema.h"
-#include "parquet/statistics.h"
-#include "parquet/types.h"
-#include "parquet/util/memory.h"
-#include "parquet/util/visibility.h"
-
-namespace parquet {
-
-using KeyValueMetadata = ::arrow::KeyValueMetadata;
-
-class ApplicationVersion {
- public:
- // Known Versions with Issues
- static const ApplicationVersion& PARQUET_251_FIXED_VERSION();
- static const ApplicationVersion& PARQUET_816_FIXED_VERSION();
- static const ApplicationVersion& PARQUET_CPP_FIXED_STATS_VERSION();
- // Regular expression for the version format
- // major . minor . patch unknown - prerelease.x + build info
- // Eg: 1.5.0ab-cdh5.5.0+cd
- static constexpr char const* VERSION_FORMAT =
- "^(\\d+)\\.(\\d+)\\.(\\d+)([^-+]*)?(?:-([^+]*))?(?:\\+(.*))?$";
- // Regular expression for the application format
- // application_name version VERSION_FORMAT (build build_name)
- // Eg: parquet-cpp version 1.5.0ab-xyz5.5.0+cd (build abcd)
- static constexpr char const* APPLICATION_FORMAT =
- "(.*?)\\s*(?:(version\\s*(?:([^(]*?)\\s*(?:\\(\\s*build\\s*([^)]*?)\\s*\\))?)?)?)";
-
- // Application that wrote the file. e.g. "IMPALA"
- std::string application_;
- // Build name
- std::string build_;
-
- // Version of the application that wrote the file, expressed as
- // (<major>.<minor>.<patch>). Unmatched parts default to 0.
- // "1.2.3" => {1, 2, 3}
- // "1.2" => {0, 0, 0}
- // "1.2-cdh5" => {0, 0, 0}
- // TODO (majetideepak): Implement support for pre_release
- struct {
- int major;
- int minor;
- int patch;
- std::string unknown;
- std::string pre_release;
- std::string build_info;
- } version;
-
- ApplicationVersion() {}
- explicit ApplicationVersion(const std::string& created_by);
- ApplicationVersion(const std::string& application, int major, int minor, int patch);
-
- // Returns true if version is strictly less than other_version
- bool VersionLt(const ApplicationVersion& other_version) const;
-
- // Returns true if version is strictly less than other_version
- bool VersionEq(const ApplicationVersion& other_version) const;
-
- // Checks if the Version has the correct statistics for a given column
- bool HasCorrectStatistics(Type::type primitive,
- SortOrder::type sort_order = SortOrder::SIGNED) const;
-};
-
-class PARQUET_EXPORT ColumnChunkMetaData {
- public:
- // API convenience to get a MetaData accessor
- static std::unique_ptr<ColumnChunkMetaData> Make(
- const uint8_t* metadata, const ColumnDescriptor* descr,
- const ApplicationVersion* writer_version = nullptr);
-
- ~ColumnChunkMetaData();
-
- // column chunk
- int64_t file_offset() const;
- // parameter is only used when a dataset is spread across multiple files
- const std::string& file_path() const;
- // column metadata
- Type::type type() const;
- int64_t num_values() const;
- std::shared_ptr<schema::ColumnPath> path_in_schema() const;
- bool is_stats_set() const;
- std::shared_ptr<RowGroupStatistics> statistics() const;
- Compression::type compression() const;
- const std::vector<Encoding::type>& encodings() const;
- bool has_dictionary_page() const;
- int64_t dictionary_page_offset() const;
- int64_t data_page_offset() const;
- bool has_index_page() const;
- int64_t index_page_offset() const;
- int64_t total_compressed_size() const;
- int64_t total_uncompressed_size() const;
-
- private:
- explicit ColumnChunkMetaData(const uint8_t* metadata, const ColumnDescriptor* descr,
- const ApplicationVersion* writer_version = nullptr);
- // PIMPL Idiom
- class ColumnChunkMetaDataImpl;
- std::unique_ptr<ColumnChunkMetaDataImpl> impl_;
-};
-
-class PARQUET_EXPORT RowGroupMetaData {
- public:
- // API convenience to get a MetaData accessor
- static std::unique_ptr<RowGroupMetaData> Make(
- const uint8_t* metadata, const SchemaDescriptor* schema,
- const ApplicationVersion* writer_version = nullptr);
-
- ~RowGroupMetaData();
-
- // row-group metadata
- int num_columns() const;
- int64_t num_rows() const;
- int64_t total_byte_size() const;
- // Return const-pointer to make it clear that this object is not to be copied
- const SchemaDescriptor* schema() const;
- std::unique_ptr<ColumnChunkMetaData> ColumnChunk(int i) const;
-
- private:
- explicit RowGroupMetaData(const uint8_t* metadata, const SchemaDescriptor* schema,
- const ApplicationVersion* writer_version = nullptr);
- // PIMPL Idiom
- class RowGroupMetaDataImpl;
- std::unique_ptr<RowGroupMetaDataImpl> impl_;
-};
-
-class FileMetaDataBuilder;
-
-class PARQUET_EXPORT FileMetaData {
- public:
- // API convenience to get a MetaData accessor
- static std::shared_ptr<FileMetaData> Make(const uint8_t* serialized_metadata,
- uint32_t* metadata_len);
-
- ~FileMetaData();
-
- // file metadata
- uint32_t size() const;
- int num_columns() const;
- int64_t num_rows() const;
- int num_row_groups() const;
- ParquetVersion::type version() const;
- const std::string& created_by() const;
- int num_schema_elements() const;
- std::unique_ptr<RowGroupMetaData> RowGroup(int i) const;
-
- const ApplicationVersion& writer_version() const;
-
- void WriteTo(OutputStream* dst) const;
-
- // Return const-pointer to make it clear that this object is not to be copied
- const SchemaDescriptor* schema() const;
-
- std::shared_ptr<const KeyValueMetadata> key_value_metadata() const;
-
- private:
- friend FileMetaDataBuilder;
- explicit FileMetaData(const uint8_t* serialized_metadata, uint32_t* metadata_len);
-
- // PIMPL Idiom
- FileMetaData();
- class FileMetaDataImpl;
- std::unique_ptr<FileMetaDataImpl> impl_;
-};
-
-// Builder API
-class PARQUET_EXPORT ColumnChunkMetaDataBuilder {
- public:
- // API convenience to get a MetaData reader
- static std::unique_ptr<ColumnChunkMetaDataBuilder> Make(
- const std::shared_ptr<WriterProperties>& props, const ColumnDescriptor* column,
- uint8_t* contents);
-
- ~ColumnChunkMetaDataBuilder();
-
- // column chunk
- // Used when a dataset is spread across multiple files
- void set_file_path(const std::string& path);
- // column metadata
- void SetStatistics(bool is_signed, const EncodedStatistics& stats);
- // get the column descriptor
- const ColumnDescriptor* descr() const;
- // commit the metadata
- void Finish(int64_t num_values, int64_t dictonary_page_offset,
- int64_t index_page_offset, int64_t data_page_offset,
- int64_t compressed_size, int64_t uncompressed_size, bool has_dictionary,
- bool dictionary_fallback);
-
- // For writing metadata at end of column chunk
- void WriteTo(OutputStream* sink);
-
- private:
- explicit ColumnChunkMetaDataBuilder(const std::shared_ptr<WriterProperties>& props,
- const ColumnDescriptor* column, uint8_t* contents);
- // PIMPL Idiom
- class ColumnChunkMetaDataBuilderImpl;
- std::unique_ptr<ColumnChunkMetaDataBuilderImpl> impl_;
-};
-
-class PARQUET_EXPORT RowGroupMetaDataBuilder {
- public:
- // API convenience to get a MetaData reader
- static std::unique_ptr<RowGroupMetaDataBuilder> Make(
- const std::shared_ptr<WriterProperties>& props, const SchemaDescriptor* schema_,
- uint8_t* contents);
-
- ~RowGroupMetaDataBuilder();
-
- ColumnChunkMetaDataBuilder* NextColumnChunk();
- int num_columns();
- int64_t num_rows();
- int current_column() const;
-
- void set_num_rows(int64_t num_rows);
-
- // commit the metadata
- void Finish(int64_t total_bytes_written);
-
- private:
- explicit RowGroupMetaDataBuilder(const std::shared_ptr<WriterProperties>& props,
- const SchemaDescriptor* schema_, uint8_t* contents);
- // PIMPL Idiom
- class RowGroupMetaDataBuilderImpl;
- std::unique_ptr<RowGroupMetaDataBuilderImpl> impl_;
-};
-
-class PARQUET_EXPORT FileMetaDataBuilder {
- public:
- // API convenience to get a MetaData reader
- static std::unique_ptr<FileMetaDataBuilder> Make(
- const SchemaDescriptor* schema, const std::shared_ptr<WriterProperties>& props,
- const std::shared_ptr<const KeyValueMetadata>& key_value_metadata = nullptr);
-
- ~FileMetaDataBuilder();
-
- RowGroupMetaDataBuilder* AppendRowGroup();
-
- // commit the metadata
- std::unique_ptr<FileMetaData> Finish();
-
- private:
- explicit FileMetaDataBuilder(
- const SchemaDescriptor* schema, const std::shared_ptr<WriterProperties>& props,
- const std::shared_ptr<const KeyValueMetadata>& key_value_metadata = nullptr);
- // PIMPL Idiom
- class FileMetaDataBuilderImpl;
- std::unique_ptr<FileMetaDataBuilderImpl> impl_;
-};
-
-} // namespace parquet
-
-#endif // PARQUET_FILE_METADATA_H
diff --git a/src/parquet/murmur3.cc b/src/parquet/murmur3.cc
deleted file mode 100644
index a19436e..0000000
--- a/src/parquet/murmur3.cc
+++ /dev/null
@@ -1,222 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-//-----------------------------------------------------------------------------
-// MurmurHash3 was written by Austin Appleby, and is placed in the public
-// domain. The author hereby disclaims copyright to this source code.
-
-// Note - The x86 and x64 versions do _not_ produce the same results, as the
-// algorithms are optimized for their respective platforms. You can still
-// compile and run any of them on any platform, but your performance with the
-// non-native version will be less than optimal.
-
-#include "parquet/murmur3.h"
-
-namespace parquet {
-
-#if defined(_MSC_VER)
-
-#define FORCE_INLINE __forceinline
-#define ROTL64(x, y) _rotl64(x, y)
-
-#else // defined(_MSC_VER)
-
-#define FORCE_INLINE inline __attribute__((always_inline))
-inline uint64_t rotl64(uint64_t x, int8_t r) { return (x << r) | (x >> (64 - r)); }
-#define ROTL64(x, y) rotl64(x, y)
-
-#endif // !defined(_MSC_VER)
-
-#define BIG_CONSTANT(x) (x##LLU)
-
-//-----------------------------------------------------------------------------
-// Block read - if your platform needs to do endian-swapping or can only
-// handle aligned reads, do the conversion here
-
-FORCE_INLINE uint32_t getblock32(const uint32_t* p, int i) { return p[i]; }
-
-FORCE_INLINE uint64_t getblock64(const uint64_t* p, int i) { return p[i]; }
-
-//-----------------------------------------------------------------------------
-// Finalization mix - force all bits of a hash block to avalanche
-
-FORCE_INLINE uint32_t fmix32(uint32_t h) {
- h ^= h >> 16;
- h *= 0x85ebca6b;
- h ^= h >> 13;
- h *= 0xc2b2ae35;
- h ^= h >> 16;
-
- return h;
-}
-
-//----------
-
-FORCE_INLINE uint64_t fmix64(uint64_t k) {
- k ^= k >> 33;
- k *= BIG_CONSTANT(0xff51afd7ed558ccd);
- k ^= k >> 33;
- k *= BIG_CONSTANT(0xc4ceb9fe1a85ec53);
- k ^= k >> 33;
-
- return k;
-}
-
-//-----------------------------------------------------------------------------
-
-void Hash_x64_128(const void* key, const int len, const uint32_t seed, uint64_t out[2]) {
- const uint8_t* data = (const uint8_t*)key;
- const int nblocks = len / 16;
-
- uint64_t h1 = seed;
- uint64_t h2 = seed;
-
- const uint64_t c1 = BIG_CONSTANT(0x87c37b91114253d5);
- const uint64_t c2 = BIG_CONSTANT(0x4cf5ad432745937f);
-
- //----------
- // body
-
- const uint64_t* blocks = (const uint64_t*)(data);
-
- for (int i = 0; i < nblocks; i++) {
- uint64_t k1 = getblock64(blocks, i * 2 + 0);
- uint64_t k2 = getblock64(blocks, i * 2 + 1);
-
- k1 *= c1;
- k1 = ROTL64(k1, 31);
- k1 *= c2;
- h1 ^= k1;
-
- h1 = ROTL64(h1, 27);
- h1 += h2;
- h1 = h1 * 5 + 0x52dce729;
-
- k2 *= c2;
- k2 = ROTL64(k2, 33);
- k2 *= c1;
- h2 ^= k2;
-
- h2 = ROTL64(h2, 31);
- h2 += h1;
- h2 = h2 * 5 + 0x38495ab5;
- }
-
- //----------
- // tail
-
- const uint8_t* tail = (const uint8_t*)(data + nblocks * 16);
-
- uint64_t k1 = 0;
- uint64_t k2 = 0;
-
- switch (len & 15) {
- case 15:
- k2 ^= ((uint64_t)tail[14]) << 48;
- case 14:
- k2 ^= ((uint64_t)tail[13]) << 40;
- case 13:
- k2 ^= ((uint64_t)tail[12]) << 32;
- case 12:
- k2 ^= ((uint64_t)tail[11]) << 24;
- case 11:
- k2 ^= ((uint64_t)tail[10]) << 16;
- case 10:
- k2 ^= ((uint64_t)tail[9]) << 8;
- case 9:
- k2 ^= ((uint64_t)tail[8]) << 0;
- k2 *= c2;
- k2 = ROTL64(k2, 33);
- k2 *= c1;
- h2 ^= k2;
-
- case 8:
- k1 ^= ((uint64_t)tail[7]) << 56;
- case 7:
- k1 ^= ((uint64_t)tail[6]) << 48;
- case 6:
- k1 ^= ((uint64_t)tail[5]) << 40;
- case 5:
- k1 ^= ((uint64_t)tail[4]) << 32;
- case 4:
- k1 ^= ((uint64_t)tail[3]) << 24;
- case 3:
- k1 ^= ((uint64_t)tail[2]) << 16;
- case 2:
- k1 ^= ((uint64_t)tail[1]) << 8;
- case 1:
- k1 ^= ((uint64_t)tail[0]) << 0;
- k1 *= c1;
- k1 = ROTL64(k1, 31);
- k1 *= c2;
- h1 ^= k1;
- }
-
- //----------
- // finalization
-
- h1 ^= len;
- h2 ^= len;
-
- h1 += h2;
- h2 += h1;
-
- h1 = fmix64(h1);
- h2 = fmix64(h2);
-
- h1 += h2;
- h2 += h1;
-
- reinterpret_cast<uint64_t*>(out)[0] = h1;
- reinterpret_cast<uint64_t*>(out)[1] = h2;
-}
-
-template <typename T>
-uint64_t HashHelper(T value, uint32_t seed) {
- uint64_t output[2];
- Hash_x64_128(reinterpret_cast<void*>(&value), sizeof(T), seed, output);
- return output[0];
-}
-
-uint64_t MurmurHash3::Hash(int32_t value) const { return HashHelper(value, seed_); }
-
-uint64_t MurmurHash3::Hash(int64_t value) const { return HashHelper(value, seed_); }
-
-uint64_t MurmurHash3::Hash(float value) const { return HashHelper(value, seed_); }
-
-uint64_t MurmurHash3::Hash(double value) const { return HashHelper(value, seed_); }
-
-uint64_t MurmurHash3::Hash(const FLBA* value, uint32_t len) const {
- uint64_t out[2];
- Hash_x64_128(reinterpret_cast<const void*>(value->ptr), len, seed_, out);
- return out[0];
-}
-
-uint64_t MurmurHash3::Hash(const Int96* value) const {
- uint64_t out[2];
- Hash_x64_128(reinterpret_cast<const void*>(value->value), sizeof(value->value), seed_,
- out);
- return out[0];
-}
-
-uint64_t MurmurHash3::Hash(const ByteArray* value) const {
- uint64_t out[2];
- Hash_x64_128(reinterpret_cast<const void*>(value->ptr), value->len, seed_, out);
- return out[0];
-}
-
-} // namespace parquet
diff --git a/src/parquet/murmur3.h b/src/parquet/murmur3.h
deleted file mode 100644
index 84792f3..0000000
--- a/src/parquet/murmur3.h
+++ /dev/null
@@ -1,56 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-//-----------------------------------------------------------------------------
-// MurmurHash3 was written by Austin Appleby, and is placed in the public
-// domain. The author hereby disclaims copyright to this source code.
-
-#ifndef PARQUET_MURMURHASH3_H_
-#define PARQUET_MURMURHASH3_H_
-
-#include <cstdint>
-
-#include "parquet/hasher.h"
-#include "parquet/types.h"
-
-namespace parquet {
-
-/// Source:
-/// https://github.com/aappleby/smhasher/blob/master/src/MurmurHash3.cpp
-/// (Modified to adapt to coding conventions and to inherit the Hasher abstract class)
-class MurmurHash3 : public Hasher {
- public:
- MurmurHash3() : seed_(DEFAULT_SEED) {}
- uint64_t Hash(int32_t value) const override;
- uint64_t Hash(int64_t value) const override;
- uint64_t Hash(float value) const override;
- uint64_t Hash(double value) const override;
- uint64_t Hash(const Int96* value) const override;
- uint64_t Hash(const ByteArray* value) const override;
- uint64_t Hash(const FLBA* val, uint32_t len) const override;
-
- private:
- // Default seed for hash which comes from Bloom filter in parquet-mr, it is generated
- // by System.nanoTime() of java.
- static constexpr int DEFAULT_SEED = 1361930890;
-
- uint32_t seed_;
-};
-
-} // namespace parquet
-
-#endif // PARQUET_MURMURHASH3_H_
diff --git a/src/parquet/parquet.pc.in b/src/parquet/parquet.pc.in
deleted file mode 100644
index 9313d1f..0000000
--- a/src/parquet/parquet.pc.in
+++ /dev/null
@@ -1,29 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-prefix=@CMAKE_INSTALL_PREFIX@
-libdir=${prefix}/@CMAKE_INSTALL_LIBDIR@
-includedir=${prefix}/@CMAKE_INSTALL_INCLUDEDIR@
-
-so_version=@PARQUET_SO_VERSION@
-abi_version=@PARQUET_ABI_VERSION@
-
-Name: Apache Parquet
-Description: Apache Parquet is a columnar storage format.
-Version: @PARQUET_VERSION@
-Libs: -L${libdir} -lparquet
-Cflags: -I${includedir}
diff --git a/src/parquet/parquet.thrift b/src/parquet/parquet.thrift
deleted file mode 100644
index aec8e54..0000000
--- a/src/parquet/parquet.thrift
+++ /dev/null
@@ -1,924 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-cpp_include "parquet/util/windows_compatibility.h"
-
-/**
- * File format description for the parquet file format
- */
-namespace cpp parquet.format
-namespace java org.apache.parquet.format
-
-/**
- * Types supported by Parquet. These types are intended to be used in combination
- * with the encodings to control the on disk storage format.
- * For example INT16 is not included as a type since a good encoding of INT32
- * would handle this.
- */
-enum Type {
- BOOLEAN = 0;
- INT32 = 1;
- INT64 = 2;
- INT96 = 3;
- FLOAT = 4;
- DOUBLE = 5;
- BYTE_ARRAY = 6;
- FIXED_LEN_BYTE_ARRAY = 7;
-}
-
-/**
- * Common types used by frameworks(e.g. hive, pig) using parquet. This helps map
- * between types in those frameworks to the base types in parquet. This is only
- * metadata and not needed to read or write the data.
- */
-enum ConvertedType {
- /** a BYTE_ARRAY actually contains UTF8 encoded chars */
- UTF8 = 0;
-
- /** a map is converted as an optional field containing a repeated key/value pair */
- MAP = 1;
-
- /** a key/value pair is converted into a group of two fields */
- MAP_KEY_VALUE = 2;
-
- /** a list is converted into an optional field containing a repeated field for its
- * values */
- LIST = 3;
-
- /** an enum is converted into a binary field */
- ENUM = 4;
-
- /**
- * A decimal value.
- *
- * This may be used to annotate binary or fixed primitive types. The
- * underlying byte array stores the unscaled value encoded as two's
- * complement using big-endian byte order (the most significant byte is the
- * zeroth element). The value of the decimal is the value * 10^{-scale}.
- *
- * This must be accompanied by a (maximum) precision and a scale in the
- * SchemaElement. The precision specifies the number of digits in the decimal
- * and the scale stores the location of the decimal point. For example 1.23
- * would have precision 3 (3 total digits) and scale 2 (the decimal point is
- * 2 digits over).
- */
- DECIMAL = 5;
-
- /**
- * A Date
- *
- * Stored as days since Unix epoch, encoded as the INT32 physical type.
- *
- */
- DATE = 6;
-
- /**
- * A time
- *
- * The total number of milliseconds since midnight. The value is stored
- * as an INT32 physical type.
- */
- TIME_MILLIS = 7;
-
- /**
- * A time.
- *
- * The total number of microseconds since midnight. The value is stored as
- * an INT64 physical type.
- */
- TIME_MICROS = 8;
-
- /**
- * A date/time combination
- *
- * Date and time recorded as milliseconds since the Unix epoch. Recorded as
- * a physical type of INT64.
- */
- TIMESTAMP_MILLIS = 9;
-
- /**
- * A date/time combination
- *
- * Date and time recorded as microseconds since the Unix epoch. The value is
- * stored as an INT64 physical type.
- */
- TIMESTAMP_MICROS = 10;
-
-
- /**
- * An unsigned integer value.
- *
- * The number describes the maximum number of meainful data bits in
- * the stored value. 8, 16 and 32 bit values are stored using the
- * INT32 physical type. 64 bit values are stored using the INT64
- * physical type.
- *
- */
- UINT_8 = 11;
- UINT_16 = 12;
- UINT_32 = 13;
- UINT_64 = 14;
-
- /**
- * A signed integer value.
- *
- * The number describes the maximum number of meainful data bits in
- * the stored value. 8, 16 and 32 bit values are stored using the
- * INT32 physical type. 64 bit values are stored using the INT64
- * physical type.
- *
- */
- INT_8 = 15;
- INT_16 = 16;
- INT_32 = 17;
- INT_64 = 18;
-
- /**
- * An embedded JSON document
- *
- * A JSON document embedded within a single UTF8 column.
- */
- JSON = 19;
-
- /**
- * An embedded BSON document
- *
- * A BSON document embedded within a single BINARY column.
- */
- BSON = 20;
-
- /**
- * An interval of time
- *
- * This type annotates data stored as a FIXED_LEN_BYTE_ARRAY of length 12
- * This data is composed of three separate little endian unsigned
- * integers. Each stores a component of a duration of time. The first
- * integer identifies the number of months associated with the duration,
- * the second identifies the number of days associated with the duration
- * and the third identifies the number of milliseconds associated with
- * the provided duration. This duration of time is independent of any
- * particular timezone or date.
- */
- INTERVAL = 21;
-}
-
-/**
- * Representation of Schemas
- */
-enum FieldRepetitionType {
- /** This field is required (can not be null) and each record has exactly 1 value. */
- REQUIRED = 0;
-
- /** The field is optional (can be null) and each record has 0 or 1 values. */
- OPTIONAL = 1;
-
- /** The field is repeated and can contain 0 or more values */
- REPEATED = 2;
-}
-
-/**
- * Statistics per row group and per page
- * All fields are optional.
- */
-struct Statistics {
- /**
- * DEPRECATED: min and max value of the column. Use min_value and max_value.
- *
- * Values are encoded using PLAIN encoding, except that variable-length byte
- * arrays do not include a length prefix.
- *
- * These fields encode min and max values determined by signed comparison
- * only. New files should use the correct order for a column's logical type
- * and store the values in the min_value and max_value fields.
- *
- * To support older readers, these may be set when the column order is
- * signed.
- */
- 1: optional binary max;
- 2: optional binary min;
- /** count of null value in the column */
- 3: optional i64 null_count;
- /** count of distinct values occurring */
- 4: optional i64 distinct_count;
- /**
- * Min and max values for the column, determined by its ColumnOrder.
- *
- * Values are encoded using PLAIN encoding, except that variable-length byte
- * arrays do not include a length prefix.
- */
- 5: optional binary max_value;
- 6: optional binary min_value;
-}
-
-/** Empty structs to use as logical type annotations */
-struct StringType {} // allowed for BINARY, must be encoded with UTF-8
-struct UUIDType {} // allowed for FIXED[16], must encoded raw UUID bytes
-struct MapType {} // see LogicalTypes.md
-struct ListType {} // see LogicalTypes.md
-struct EnumType {} // allowed for BINARY, must be encoded with UTF-8
-struct DateType {} // allowed for INT32
-
-/**
- * Logical type to annotate a column that is always null.
- *
- * Sometimes when discovering the schema of existing data, values are always
- * null and the physical type can't be determined. This annotation signals
- * the case where the physical type was guessed from all null values.
- */
-struct NullType {} // allowed for any physical type, only null values stored
-
-/**
- * Decimal logical type annotation
- *
- * To maintain forward-compatibility in v1, implementations using this logical
- * type must also set scale and precision on the annotated SchemaElement.
- *
- * Allowed for physical types: INT32, INT64, FIXED, and BINARY
- */
-struct DecimalType {
- 1: required i32 scale
- 2: required i32 precision
-}
-
-/** Time units for logical types */
-struct MilliSeconds {}
-struct MicroSeconds {}
-union TimeUnit {
- 1: MilliSeconds MILLIS
- 2: MicroSeconds MICROS
-}
-
-/**
- * Timestamp logical type annotation
- *
- * Allowed for physical types: INT64
- */
-struct TimestampType {
- 1: required bool isAdjustedToUTC
- 2: required TimeUnit unit
-}
-
-/**
- * Time logical type annotation
- *
- * Allowed for physical types: INT32 (millis), INT64 (micros)
- */
-struct TimeType {
- 1: required bool isAdjustedToUTC
- 2: required TimeUnit unit
-}
-
-/**
- * Integer logical type annotation
- *
- * bitWidth must be 8, 16, 32, or 64.
- *
- * Allowed for physical types: INT32, INT64
- */
-struct IntType {
- 1: required byte bitWidth
- 2: required bool isSigned
-}
-
-/**
- * Embedded JSON logical type annotation
- *
- * Allowed for physical types: BINARY
- */
-struct JsonType {
-}
-
-/**
- * Embedded BSON logical type annotation
- *
- * Allowed for physical types: BINARY
- */
-struct BsonType {
-}
-
-/**
- * LogicalType annotations to replace ConvertedType.
- *
- * To maintain compatibility, implementations using LogicalType for a
- * SchemaElement must also set the corresponding ConvertedType from the
- * following table.
- */
-union LogicalType {
- 1: StringType STRING // use ConvertedType UTF8 if encoding is UTF-8
- 2: MapType MAP // use ConvertedType MAP
- 3: ListType LIST // use ConvertedType LIST
- 4: EnumType ENUM // use ConvertedType ENUM
- 5: DecimalType DECIMAL // use ConvertedType DECIMAL
- 6: DateType DATE // use ConvertedType DATE
- 7: TimeType TIME // use ConvertedType TIME_MICROS or TIME_MILLIS
- 8: TimestampType TIMESTAMP // use ConvertedType TIMESTAMP_MICROS or TIMESTAMP_MILLIS
- // 9: reserved for INTERVAL
- 10: IntType INTEGER // use ConvertedType INT_* or UINT_*
- 11: NullType UNKNOWN // no compatible ConvertedType
- 12: JsonType JSON // use ConvertedType JSON
- 13: BsonType BSON // use ConvertedType BSON
-}
-
-/**
- * Represents a element inside a schema definition.
- * - if it is a group (inner node) then type is undefined and num_children is defined
- * - if it is a primitive type (leaf) then type is defined and num_children is undefined
- * the nodes are listed in depth first traversal order.
- */
-struct SchemaElement {
- /** Data type for this field. Not set if the current element is a non-leaf node */
- 1: optional Type type;
-
- /** If type is FIXED_LEN_BYTE_ARRAY, this is the byte length of the vales.
- * Otherwise, if specified, this is the maximum bit length to store any of the values.
- * (e.g. a low cardinality INT col could have this set to 3). Note that this is
- * in the schema, and therefore fixed for the entire file.
- */
- 2: optional i32 type_length;
-
- /** repetition of the field. The root of the schema does not have a repetition_type.
- * All other nodes must have one */
- 3: optional FieldRepetitionType repetition_type;
-
- /** Name of the field in the schema */
- 4: required string name;
-
- /** Nested fields. Since thrift does not support nested fields,
- * the nesting is flattened to a single list by a depth-first traversal.
- * The children count is used to construct the nested relationship.
- * This field is not set when the element is a primitive type
- */
- 5: optional i32 num_children;
-
- /** When the schema is the result of a conversion from another model
- * Used to record the original type to help with cross conversion.
- */
- 6: optional ConvertedType converted_type;
-
- /** Used when this column contains decimal data.
- * See the DECIMAL converted type for more details.
- */
- 7: optional i32 scale
- 8: optional i32 precision
-
- /** When the original schema supports field ids, this will save the
- * original field id in the parquet schema
- */
- 9: optional i32 field_id;
-
- /**
- * The logical type of this SchemaElement; only valid for primitives.
- *
- * LogicalType replaces ConvertedType, but ConvertedType is still required
- * for some logical types to ensure forward-compatibility in format v1.
- */
- 10: optional LogicalType logicalType
-}
-
-/**
- * Encodings supported by Parquet. Not all encodings are valid for all types. These
- * enums are also used to specify the encoding of definition and repetition levels.
- * See the accompanying doc for the details of the more complicated encodings.
- */
-enum Encoding {
- /** Default encoding.
- * BOOLEAN - 1 bit per value. 0 is false; 1 is true.
- * INT32 - 4 bytes per value. Stored as little-endian.
- * INT64 - 8 bytes per value. Stored as little-endian.
- * FLOAT - 4 bytes per value. IEEE. Stored as little-endian.
- * DOUBLE - 8 bytes per value. IEEE. Stored as little-endian.
- * BYTE_ARRAY - 4 byte length stored as little endian, followed by bytes.
- * FIXED_LEN_BYTE_ARRAY - Just the bytes.
- */
- PLAIN = 0;
-
- /** Group VarInt encoding for INT32/INT64.
- * This encoding is deprecated. It was never used
- */
- // GROUP_VAR_INT = 1;
-
- /**
- * Deprecated: Dictionary encoding. The values in the dictionary are encoded in the
- * plain type.
- * in a data page use RLE_DICTIONARY instead.
- * in a Dictionary page use PLAIN instead
- */
- PLAIN_DICTIONARY = 2;
-
- /** Group packed run length encoding. Usable for definition/repetition levels
- * encoding and Booleans (on one bit: 0 is false; 1 is true.)
- */
- RLE = 3;
-
- /** Bit packed encoding. This can only be used if the data has a known max
- * width. Usable for definition/repetition levels encoding.
- */
- BIT_PACKED = 4;
-
- /** Delta encoding for integers. This can be used for int columns and works best
- * on sorted data
- */
- DELTA_BINARY_PACKED = 5;
-
- /** Encoding for byte arrays to separate the length values and the data. The lengths
- * are encoded using DELTA_BINARY_PACKED
- */
- DELTA_LENGTH_BYTE_ARRAY = 6;
-
- /** Incremental-encoded byte array. Prefix lengths are encoded using DELTA_BINARY_PACKED.
- * Suffixes are stored as delta length byte arrays.
- */
- DELTA_BYTE_ARRAY = 7;
-
- /** Dictionary encoding: the ids are encoded using the RLE encoding
- */
- RLE_DICTIONARY = 8;
-}
-
-/**
- * Supported compression algorithms.
- *
- * Codecs added in 2.3.2 can be read by readers based on 2.3.2 and later.
- * Codec support may vary between readers based on the format version and
- * libraries available at runtime. Gzip, Snappy, and LZ4 codecs are
- * widely available, while Zstd and Brotli require additional libraries.
- */
-enum CompressionCodec {
- UNCOMPRESSED = 0;
- SNAPPY = 1;
- GZIP = 2;
- LZO = 3;
- BROTLI = 4; // Added in 2.3.2
- LZ4 = 5; // Added in 2.3.2
- ZSTD = 6; // Added in 2.3.2
-}
-
-enum PageType {
- DATA_PAGE = 0;
- INDEX_PAGE = 1;
- DICTIONARY_PAGE = 2;
- DATA_PAGE_V2 = 3;
-}
-
-/**
- * Enum to annotate whether lists of min/max elements inside ColumnIndex
- * are ordered and if so, in which direction.
- */
-enum BoundaryOrder {
- UNORDERED = 0;
- ASCENDING = 1;
- DESCENDING = 2;
-}
-
-/** Data page header */
-struct DataPageHeader {
- /** Number of values, including NULLs, in this data page. **/
- 1: required i32 num_values
-
- /** Encoding used for this data page **/
- 2: required Encoding encoding
-
- /** Encoding used for definition levels **/
- 3: required Encoding definition_level_encoding;
-
- /** Encoding used for repetition levels **/
- 4: required Encoding repetition_level_encoding;
-
- /** Optional statistics for the data in this page**/
- 5: optional Statistics statistics;
-}
-
-struct IndexPageHeader {
- /** TODO: **/
-}
-
-struct DictionaryPageHeader {
- /** Number of values in the dictionary **/
- 1: required i32 num_values;
-
- /** Encoding using this dictionary page **/
- 2: required Encoding encoding
-
- /** If true, the entries in the dictionary are sorted in ascending order **/
- 3: optional bool is_sorted;
-}
-
-/**
- * New page format allowing reading levels without decompressing the data
- * Repetition and definition levels are uncompressed
- * The remaining section containing the data is compressed if is_compressed is true
- **/
-struct DataPageHeaderV2 {
- /** Number of values, including NULLs, in this data page. **/
- 1: required i32 num_values
- /** Number of NULL values, in this data page.
- Number of non-null = num_values - num_nulls which is also the number of values in the data section **/
- 2: required i32 num_nulls
- /** Number of rows in this data page. which means pages change on record boundaries (r = 0) **/
- 3: required i32 num_rows
- /** Encoding used for data in this page **/
- 4: required Encoding encoding
-
- // repetition levels and definition levels are always using RLE (without size in it)
-
- /** length of the definition levels */
- 5: required i32 definition_levels_byte_length;
- /** length of the repetition levels */
- 6: required i32 repetition_levels_byte_length;
-
- /** whether the values are compressed.
- Which means the section of the page between
- definition_levels_byte_length + repetition_levels_byte_length + 1 and compressed_page_size (included)
- is compressed with the compression_codec.
- If missing it is considered compressed */
- 7: optional bool is_compressed = 1;
-
- /** optional statistics for this column chunk */
- 8: optional Statistics statistics;
-}
-
-struct PageHeader {
- /** the type of the page: indicates which of the *_header fields is set **/
- 1: required PageType type
-
- /** Uncompressed page size in bytes (not including this header) **/
- 2: required i32 uncompressed_page_size
-
- /** Compressed page size in bytes (not including this header) **/
- 3: required i32 compressed_page_size
-
- /** 32bit crc for the data below. This allows for disabling checksumming in HDFS
- * if only a few pages needs to be read
- **/
- 4: optional i32 crc
-
- // Headers for page specific data. One only will be set.
- 5: optional DataPageHeader data_page_header;
- 6: optional IndexPageHeader index_page_header;
- 7: optional DictionaryPageHeader dictionary_page_header;
- 8: optional DataPageHeaderV2 data_page_header_v2;
-}
-
-/**
- * Wrapper struct to store key values
- */
- struct KeyValue {
- 1: required string key
- 2: optional string value
-}
-
-/**
- * Wrapper struct to specify sort order
- */
-struct SortingColumn {
- /** The column index (in this row group) **/
- 1: required i32 column_idx
-
- /** If true, indicates this column is sorted in descending order. **/
- 2: required bool descending
-
- /** If true, nulls will come before non-null values, otherwise,
- * nulls go at the end. */
- 3: required bool nulls_first
-}
-
-/**
- * statistics of a given page type and encoding
- */
-struct PageEncodingStats {
-
- /** the page type (data/dic/...) **/
- 1: required PageType page_type;
-
- /** encoding of the page **/
- 2: required Encoding encoding;
-
- /** number of pages of this type with this encoding **/
- 3: required i32 count;
-
-}
-
-/**
- * Description for column metadata
- */
-struct ColumnMetaData {
- /** Type of this column **/
- 1: required Type type
-
- /** Set of all encodings used for this column. The purpose is to validate
- * whether we can decode those pages. **/
- 2: required list<Encoding> encodings
-
- /** Path in schema **/
- 3: required list<string> path_in_schema
-
- /** Compression codec **/
- 4: required CompressionCodec codec
-
- /** Number of values in this column **/
- 5: required i64 num_values
-
- /** total byte size of all uncompressed pages in this column chunk (including the headers) **/
- 6: required i64 total_uncompressed_size
-
- /** total byte size of all compressed pages in this column chunk (including the headers) **/
- 7: required i64 total_compressed_size
-
- /** Optional key/value metadata **/
- 8: optional list<KeyValue> key_value_metadata
-
- /** Byte offset from beginning of file to first data page **/
- 9: required i64 data_page_offset
-
- /** Byte offset from beginning of file to root index page **/
- 10: optional i64 index_page_offset
-
- /** Byte offset from the beginning of file to first (only) dictionary page **/
- 11: optional i64 dictionary_page_offset
-
- /** optional statistics for this column chunk */
- 12: optional Statistics statistics;
-
- /** Set of all encodings used for pages in this column chunk.
- * This information can be used to determine if all data pages are
- * dictionary encoded for example **/
- 13: optional list<PageEncodingStats> encoding_stats;
-}
-
-struct EncryptionWithFooterKey {
-}
-
-struct EncryptionWithColumnKey {
- /** Column path in schema **/
- 1: required list<string> path_in_schema
-
- /** Retrieval metadata of the column-specific key **/
- 2: optional binary column_key_metadata
-}
-
-union ColumnCryptoMetaData {
- 1: EncryptionWithFooterKey ENCRYPTION_WITH_FOOTER_KEY
- 2: EncryptionWithColumnKey ENCRYPTION_WITH_COLUMN_KEY
-}
-
-struct ColumnChunk {
- /** File where column data is stored. If not set, assumed to be same file as
- * metadata. This path is relative to the current file.
- **/
- 1: optional string file_path
-
- /** Byte offset in file_path to the ColumnMetaData **/
- 2: required i64 file_offset
-
- /** Column metadata for this chunk. This is the same content as what is at
- * file_path/file_offset. Having it here has it replicated in the file
- * metadata.
- **/
- 3: optional ColumnMetaData meta_data
-
- /** File offset of ColumnChunk's OffsetIndex **/
- 4: optional i64 offset_index_offset
-
- /** Size of ColumnChunk's OffsetIndex, in bytes **/
- 5: optional i32 offset_index_length
-
- /** File offset of ColumnChunk's ColumnIndex **/
- 6: optional i64 column_index_offset
-
- /** Size of ColumnChunk's ColumnIndex, in bytes **/
- 7: optional i32 column_index_length
-
- /** Crypto metadata of encrypted columns **/
- 8: optional ColumnCryptoMetaData crypto_meta_data
-}
-
-struct RowGroup {
- /** Metadata for each column chunk in this row group.
- * This list must have the same order as the SchemaElement list in FileMetaData.
- **/
- 1: required list<ColumnChunk> columns
-
- /** Total byte size of all the uncompressed column data in this row group **/
- 2: required i64 total_byte_size
-
- /** Number of rows in this row group **/
- 3: required i64 num_rows
-
- /** If set, specifies a sort ordering of the rows in this RowGroup.
- * The sorting columns can be a subset of all the columns.
- */
- 4: optional list<SortingColumn> sorting_columns
-}
-
-/** Empty struct to signal the order defined by the physical or logical type */
-struct TypeDefinedOrder {}
-
-/**
- * Union to specify the order used for the min_value and max_value fields for a
- * column. This union takes the role of an enhanced enum that allows rich
- * elements (which will be needed for a collation-based ordering in the future).
- *
- * Possible values are:
- * * TypeDefinedOrder - the column uses the order defined by its logical or
- * physical type (if there is no logical type).
- *
- * If the reader does not support the value of this union, min and max stats
- * for this column should be ignored.
- */
-union ColumnOrder {
-
- /**
- * The sort orders for logical types are:
- * UTF8 - unsigned byte-wise comparison
- * INT8 - signed comparison
- * INT16 - signed comparison
- * INT32 - signed comparison
- * INT64 - signed comparison
- * UINT8 - unsigned comparison
- * UINT16 - unsigned comparison
- * UINT32 - unsigned comparison
- * UINT64 - unsigned comparison
- * DECIMAL - signed comparison of the represented value
- * DATE - signed comparison
- * TIME_MILLIS - signed comparison
- * TIME_MICROS - signed comparison
- * TIMESTAMP_MILLIS - signed comparison
- * TIMESTAMP_MICROS - signed comparison
- * INTERVAL - unsigned comparison
- * JSON - unsigned byte-wise comparison
- * BSON - unsigned byte-wise comparison
- * ENUM - unsigned byte-wise comparison
- * LIST - undefined
- * MAP - undefined
- *
- * In the absence of logical types, the sort order is determined by the physical type:
- * BOOLEAN - false, true
- * INT32 - signed comparison
- * INT64 - signed comparison
- * INT96 (only used for legacy timestamps) - unsigned comparison
- * FLOAT - signed comparison of the represented value
- * DOUBLE - signed comparison of the represented value
- * BYTE_ARRAY - unsigned byte-wise comparison
- * FIXED_LEN_BYTE_ARRAY - unsigned byte-wise comparison
- */
- 1: TypeDefinedOrder TYPE_ORDER;
-}
-
-struct PageLocation {
- /** Offset of the page in the file **/
- 1: required i64 offset
-
- /**
- * Size of the page, including header. Sum of compressed_page_size and header
- * length
- */
- 2: required i32 compressed_page_size
-
- /**
- * Index within the RowGroup of the first row of the page; this means pages
- * change on record boundaries (r = 0).
- */
- 3: required i64 first_row_index
-}
-
-struct OffsetIndex {
- /**
- * PageLocations, ordered by increasing PageLocation.offset. It is required
- * that page_locations[i].first_row_index < page_locations[i+1].first_row_index.
- */
- 1: required list<PageLocation> page_locations
-}
-
-/**
- * Description for ColumnIndex.
- * Each <array-field>[i] refers to the page at OffsetIndex.page_locations[i]
- */
-struct ColumnIndex {
- /**
- * A list of Boolean values to determine the validity of the corresponding
- * min and max values. If true, a page contains only null values, and writers
- * have to set the corresponding entries in min_values and max_values to
- * byte[0], so that all lists have the same length. If false, the
- * corresponding entries in min_values and max_values must be valid.
- */
- 1: required list<bool> null_pages
-
- /**
- * Two lists containing lower and upper bounds for the values of each page.
- * These may be the actual minimum and maximum values found on a page, but
- * can also be (more compact) values that do not exist on a page. For
- * example, instead of storing ""Blart Versenwald III", a writer may set
- * min_values[i]="B", max_values[i]="C". Such more compact values must still
- * be valid values within the column's logical type. Readers must make sure
- * that list entries are populated before using them by inspecting null_pages.
- */
- 2: required list<binary> min_values
- 3: required list<binary> max_values
-
- /**
- * Stores whether both min_values and max_values are orderd and if so, in
- * which direction. This allows readers to perform binary searches in both
- * lists. Readers cannot assume that max_values[i] <= min_values[i+1], even
- * if the lists are ordered.
- */
- 4: required BoundaryOrder boundary_order
-
- /** A list containing the number of null values for each page **/
- 5: optional list<i64> null_counts
-}
-
-/**
- * Description for file metadata
- */
-struct FileMetaData {
- /** Version of this file **/
- 1: required i32 version
-
- /** Parquet schema for this file. This schema contains metadata for all the columns.
- * The schema is represented as a tree with a single root. The nodes of the tree
- * are flattened to a list by doing a depth-first traversal.
- * The column metadata contains the path in the schema for that column which can be
- * used to map columns to nodes in the schema.
- * The first element is the root **/
- 2: required list<SchemaElement> schema;
-
- /** Number of rows in this file **/
- 3: required i64 num_rows
-
- /** Row groups in this file **/
- 4: required list<RowGroup> row_groups
-
- /** Optional key/value metadata **/
- 5: optional list<KeyValue> key_value_metadata
-
- /** String for application that wrote this file. This should be in the format
- * <Application> version <App Version> (build <App Build Hash>).
- * e.g. impala version 1.0 (build 6cf94d29b2b7115df4de2c06e2ab4326d721eb55)
- **/
- 6: optional string created_by
-
- /**
- * Sort order used for the min_value and max_value fields of each column in
- * this file. Each sort order corresponds to one column, determined by its
- * position in the list, matching the position of the column in the schema.
- *
- * Without column_orders, the meaning of the min_value and max_value fields is
- * undefined. To ensure well-defined behaviour, if min_value and max_value are
- * written to a Parquet file, column_orders must be written as well.
- *
- * The obsolete min and max fields are always sorted by signed comparison
- * regardless of column_orders.
- */
- 7: optional list<ColumnOrder> column_orders;
-}
-
-struct AesGcmV1 {
- /** Retrieval metadata of AAD used for encryption of pages and structures **/
- 1: optional binary aad_metadata
-}
-
-struct AesGcmCtrV1 {
- /** Retrieval metadata of AAD used for encryption of structures **/
- 1: optional binary aad_metadata
-}
-
-union EncryptionAlgorithm {
- 1: AesGcmV1 AES_GCM_V1
- 2: AesGcmCtrV1 AES_GCM_CTR_V1
-}
-
-struct FileCryptoMetaData {
- 1: required EncryptionAlgorithm encryption_algorithm
-
- /** Parquet footer can be encrypted, or left as plaintext **/
- 2: required bool encrypted_footer
-
- /** Retrieval metadata of key used for encryption of footer,
- * and (possibly) columns **/
- 3: optional binary footer_key_metadata
-
- /** Offset of Parquet footer (encrypted, or plaintext) **/
- 4: required i64 footer_offset
-
- /** If file IVs are comprised of a fixed part,
- * and variable parts (random or counter), keep the fixed
- * part here **/
- 5: optional binary iv_prefix
-}
-
diff --git a/src/parquet/parquet_version.h.in b/src/parquet/parquet_version.h.in
deleted file mode 100644
index db8f396..0000000
--- a/src/parquet/parquet_version.h.in
+++ /dev/null
@@ -1,24 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#ifndef PARQUET_VERSION_H
-#define PARQUET_VERSION_H
-
-// define the parquet created by version
-#define CREATED_BY_VERSION "parquet-cpp version @PARQUET_VERSION@"
-
-#endif // PARQUET_VERSION_H
diff --git a/src/parquet/printer.cc b/src/parquet/printer.cc
deleted file mode 100644
index 9f26a41..0000000
--- a/src/parquet/printer.cc
+++ /dev/null
@@ -1,254 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#include "parquet/printer.h"
-
-#include <string>
-#include <vector>
-
-#include "parquet/column_scanner.h"
-
-using std::string;
-using std::vector;
-
-namespace parquet {
-// ----------------------------------------------------------------------
-// ParquetFilePrinter::DebugPrint
-
-// the fixed initial size is just for an example
-#define COL_WIDTH "30"
-
-void ParquetFilePrinter::DebugPrint(std::ostream& stream, std::list<int> selected_columns,
- bool print_values, bool print_key_value_metadata,
- const char* filename) {
- const FileMetaData* file_metadata = fileReader->metadata().get();
-
- stream << "File Name: " << filename << "\n";
- stream << "Version: " << file_metadata->version() << "\n";
- stream << "Created By: " << file_metadata->created_by() << "\n";
- stream << "Total rows: " << file_metadata->num_rows() << "\n";
-
- if (print_key_value_metadata) {
- auto key_value_metadata = file_metadata->key_value_metadata();
- int64_t size_of_key_value_metadata = key_value_metadata->size();
- stream << "Key Value File Metadata: " << size_of_key_value_metadata << " entries\n";
- for (int64_t i = 0; i < size_of_key_value_metadata; i++) {
- stream << " Key nr " << i << " " << key_value_metadata->key(i) << ": "
- << key_value_metadata->value(i) << "\n";
- }
- }
-
- stream << "Number of RowGroups: " << file_metadata->num_row_groups() << "\n";
- stream << "Number of Real Columns: "
- << file_metadata->schema()->group_node()->field_count() << "\n";
-
- if (selected_columns.size() == 0) {
- for (int i = 0; i < file_metadata->num_columns(); i++) {
- selected_columns.push_back(i);
- }
- } else {
- for (auto i : selected_columns) {
- if (i < 0 || i >= file_metadata->num_columns()) {
- throw ParquetException("Selected column is out of range");
- }
- }
- }
-
- stream << "Number of Columns: " << file_metadata->num_columns() << "\n";
- stream << "Number of Selected Columns: " << selected_columns.size() << "\n";
- for (auto i : selected_columns) {
- const ColumnDescriptor* descr = file_metadata->schema()->Column(i);
- stream << "Column " << i << ": " << descr->name() << " ("
- << TypeToString(descr->physical_type()) << ")" << std::endl;
- }
-
- for (int r = 0; r < file_metadata->num_row_groups(); ++r) {
- stream << "--- Row Group " << r << " ---\n";
-
- auto group_reader = fileReader->RowGroup(r);
- std::unique_ptr<RowGroupMetaData> group_metadata = file_metadata->RowGroup(r);
-
- stream << "--- Total Bytes " << group_metadata->total_byte_size() << " ---\n";
- stream << " Rows: " << group_metadata->num_rows() << "---\n";
-
- // Print column metadata
- for (auto i : selected_columns) {
- auto column_chunk = group_metadata->ColumnChunk(i);
- std::shared_ptr<RowGroupStatistics> stats = column_chunk->statistics();
-
- const ColumnDescriptor* descr = file_metadata->schema()->Column(i);
- stream << "Column " << i << std::endl << ", Values: " << column_chunk->num_values();
- if (column_chunk->is_stats_set()) {
- std::string min = stats->EncodeMin(), max = stats->EncodeMax();
- stream << ", Null Values: " << stats->null_count()
- << ", Distinct Values: " << stats->distinct_count() << std::endl
- << " Max: " << FormatStatValue(descr->physical_type(), max)
- << ", Min: " << FormatStatValue(descr->physical_type(), min);
- } else {
- stream << " Statistics Not Set";
- }
- stream << std::endl
- << " Compression: " << CompressionToString(column_chunk->compression())
- << ", Encodings: ";
- for (auto encoding : column_chunk->encodings()) {
- stream << EncodingToString(encoding) << " ";
- }
- stream << std::endl
- << " Uncompressed Size: " << column_chunk->total_uncompressed_size()
- << ", Compressed Size: " << column_chunk->total_compressed_size()
- << std::endl;
- }
-
- if (!print_values) {
- continue;
- }
-
- static constexpr int bufsize = 25;
- char buffer[bufsize];
-
- // Create readers for selected columns and print contents
- vector<std::shared_ptr<Scanner>> scanners(selected_columns.size(), nullptr);
- int j = 0;
- for (auto i : selected_columns) {
- std::shared_ptr<ColumnReader> col_reader = group_reader->Column(i);
-
- std::stringstream ss;
- ss << "%-" << COL_WIDTH << "s";
- std::string fmt = ss.str();
-
- snprintf(buffer, bufsize, fmt.c_str(),
- file_metadata->schema()->Column(i)->name().c_str());
- stream << buffer;
-
- // This is OK in this method as long as the RowGroupReader does not get
- // deleted
- scanners[j++] = Scanner::Make(col_reader);
- }
- stream << "\n";
-
- bool hasRow;
- do {
- hasRow = false;
- for (auto scanner : scanners) {
- if (scanner->HasNext()) {
- hasRow = true;
- scanner->PrintNext(stream, 27);
- }
- }
- stream << "\n";
- } while (hasRow);
- }
-}
-
-void ParquetFilePrinter::JSONPrint(std::ostream& stream, std::list<int> selected_columns,
- const char* filename) {
- const FileMetaData* file_metadata = fileReader->metadata().get();
- stream << "{\n";
- stream << " \"FileName\": \"" << filename << "\",\n";
- stream << " \"Version\": \"" << file_metadata->version() << "\",\n";
- stream << " \"CreatedBy\": \"" << file_metadata->created_by() << "\",\n";
- stream << " \"TotalRows\": \"" << file_metadata->num_rows() << "\",\n";
- stream << " \"NumberOfRowGroups\": \"" << file_metadata->num_row_groups() << "\",\n";
- stream << " \"NumberOfRealColumns\": \""
- << file_metadata->schema()->group_node()->field_count() << "\",\n";
- stream << " \"NumberOfColumns\": \"" << file_metadata->num_columns() << "\",\n";
-
- if (selected_columns.size() == 0) {
- for (int i = 0; i < file_metadata->num_columns(); i++) {
- selected_columns.push_back(i);
- }
- } else {
- for (auto i : selected_columns) {
- if (i < 0 || i >= file_metadata->num_columns()) {
- throw ParquetException("Selected column is out of range");
- }
- }
- }
-
- stream << " \"Columns\": [\n";
- int c = 0;
- for (auto i : selected_columns) {
- const ColumnDescriptor* descr = file_metadata->schema()->Column(i);
- stream << " { \"Id\": \"" << i << "\", \"Name\": \"" << descr->name() << "\","
- << " \"PhysicalType\": \"" << TypeToString(descr->physical_type()) << "\","
- << " \"LogicalType\": \"" << LogicalTypeToString(descr->logical_type())
- << "\" }";
- c++;
- if (c != static_cast<int>(selected_columns.size())) {
- stream << ",\n";
- }
- }
-
- stream << "\n ],\n \"RowGroups\": [\n";
- for (int r = 0; r < file_metadata->num_row_groups(); ++r) {
- stream << " {\n \"Id\": \"" << r << "\", ";
-
- auto group_reader = fileReader->RowGroup(r);
- std::unique_ptr<RowGroupMetaData> group_metadata = file_metadata->RowGroup(r);
-
- stream << " \"TotalBytes\": \"" << group_metadata->total_byte_size() << "\", ";
- stream << " \"Rows\": \"" << group_metadata->num_rows() << "\",\n";
-
- // Print column metadata
- stream << " \"ColumnChunks\": [\n";
- int c1 = 0;
- for (auto i : selected_columns) {
- auto column_chunk = group_metadata->ColumnChunk(i);
- std::shared_ptr<RowGroupStatistics> stats = column_chunk->statistics();
-
- const ColumnDescriptor* descr = file_metadata->schema()->Column(i);
- stream << " {\"Id\": \"" << i << "\", \"Values\": \""
- << column_chunk->num_values() << "\", "
- << "\"StatsSet\": ";
- if (column_chunk->is_stats_set()) {
- stream << "\"True\", \"Stats\": {";
- std::string min = stats->EncodeMin(), max = stats->EncodeMax();
- stream << "\"NumNulls\": \"" << stats->null_count() << "\", "
- << "\"DistinctValues\": \"" << stats->distinct_count() << "\", "
- << "\"Max\": \"" << FormatStatValue(descr->physical_type(), max) << "\", "
- << "\"Min\": \"" << FormatStatValue(descr->physical_type(), min)
- << "\" },";
- } else {
- stream << "\"False\",";
- }
- stream << "\n \"Compression\": \""
- << CompressionToString(column_chunk->compression())
- << "\", \"Encodings\": \"";
- for (auto encoding : column_chunk->encodings()) {
- stream << EncodingToString(encoding) << " ";
- }
- stream << "\", "
- << "\"UncompressedSize\": \"" << column_chunk->total_uncompressed_size()
- << "\", \"CompressedSize\": \"" << column_chunk->total_compressed_size();
-
- // end of a ColumnChunk
- stream << "\" }";
- c1++;
- if (c1 != static_cast<int>(selected_columns.size())) {
- stream << ",\n";
- }
- }
-
- stream << "\n ]\n }";
- if ((r + 1) != static_cast<int>(file_metadata->num_row_groups())) {
- stream << ",\n";
- }
- }
- stream << "\n ]\n}\n";
-}
-
-} // namespace parquet
diff --git a/src/parquet/printer.h b/src/parquet/printer.h
deleted file mode 100644
index 1113c3f..0000000
--- a/src/parquet/printer.h
+++ /dev/null
@@ -1,50 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#ifndef PARQUET_FILE_PRINTER_H
-#define PARQUET_FILE_PRINTER_H
-
-#include <cstdint>
-#include <iosfwd>
-#include <list>
-#include <memory>
-#include <string>
-#include <vector>
-
-#include "parquet/file_reader.h"
-
-namespace parquet {
-
-class PARQUET_EXPORT ParquetFilePrinter {
- private:
- ParquetFileReader* fileReader;
-
- public:
- explicit ParquetFilePrinter(ParquetFileReader* reader) : fileReader(reader) {}
- ~ParquetFilePrinter() {}
-
- void DebugPrint(std::ostream& stream, std::list<int> selected_columns,
- bool print_values = true, bool print_key_value_metadata = false,
- const char* filename = "No Name");
-
- void JSONPrint(std::ostream& stream, std::list<int> selected_columns,
- const char* filename = "No Name");
-};
-
-} // namespace parquet
-
-#endif // PARQUET_FILE_PRINTER_H
diff --git a/src/parquet/properties-test.cc b/src/parquet/properties-test.cc
deleted file mode 100644
index c740b59..0000000
--- a/src/parquet/properties-test.cc
+++ /dev/null
@@ -1,66 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#include <gtest/gtest.h>
-
-#include <string>
-
-#include "parquet/file_reader.h"
-#include "parquet/properties.h"
-
-namespace parquet {
-
-using schema::ColumnPath;
-
-namespace test {
-
-TEST(TestReaderProperties, Basics) {
- ReaderProperties props;
-
- ASSERT_EQ(DEFAULT_BUFFER_SIZE, props.buffer_size());
- ASSERT_EQ(DEFAULT_USE_BUFFERED_STREAM, props.is_buffered_stream_enabled());
-}
-
-TEST(TestWriterProperties, Basics) {
- std::shared_ptr<WriterProperties> props = WriterProperties::Builder().build();
-
- ASSERT_EQ(DEFAULT_PAGE_SIZE, props->data_pagesize());
- ASSERT_EQ(DEFAULT_DICTIONARY_PAGE_SIZE_LIMIT, props->dictionary_pagesize_limit());
- ASSERT_EQ(DEFAULT_WRITER_VERSION, props->version());
-}
-
-TEST(TestWriterProperties, AdvancedHandling) {
- WriterProperties::Builder builder;
- builder.compression("gzip", Compression::GZIP);
- builder.compression("zstd", Compression::ZSTD);
- builder.compression(Compression::SNAPPY);
- builder.encoding(Encoding::DELTA_BINARY_PACKED);
- builder.encoding("delta-length", Encoding::DELTA_LENGTH_BYTE_ARRAY);
- std::shared_ptr<WriterProperties> props = builder.build();
-
- ASSERT_EQ(Compression::GZIP, props->compression(ColumnPath::FromDotString("gzip")));
- ASSERT_EQ(Compression::ZSTD, props->compression(ColumnPath::FromDotString("zstd")));
- ASSERT_EQ(Compression::SNAPPY,
- props->compression(ColumnPath::FromDotString("delta-length")));
- ASSERT_EQ(Encoding::DELTA_BINARY_PACKED,
- props->encoding(ColumnPath::FromDotString("gzip")));
- ASSERT_EQ(Encoding::DELTA_LENGTH_BYTE_ARRAY,
- props->encoding(ColumnPath::FromDotString("delta-length")));
-}
-
-} // namespace test
-} // namespace parquet
diff --git a/src/parquet/properties.h b/src/parquet/properties.h
deleted file mode 100644
index 83dc205..0000000
--- a/src/parquet/properties.h
+++ /dev/null
@@ -1,438 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#ifndef PARQUET_COLUMN_PROPERTIES_H
-#define PARQUET_COLUMN_PROPERTIES_H
-
-#include <memory>
-#include <string>
-#include <unordered_map>
-
-#include "parquet/exception.h"
-#include "parquet/parquet_version.h"
-#include "parquet/schema.h"
-#include "parquet/types.h"
-#include "parquet/util/memory.h"
-#include "parquet/util/visibility.h"
-
-namespace parquet {
-
-struct ParquetVersion {
- enum type { PARQUET_1_0, PARQUET_2_0 };
-};
-
-static int64_t DEFAULT_BUFFER_SIZE = 0;
-static bool DEFAULT_USE_BUFFERED_STREAM = false;
-
-class PARQUET_EXPORT ReaderProperties {
- public:
- explicit ReaderProperties(::arrow::MemoryPool* pool = ::arrow::default_memory_pool())
- : pool_(pool) {
- buffered_stream_enabled_ = DEFAULT_USE_BUFFERED_STREAM;
- buffer_size_ = DEFAULT_BUFFER_SIZE;
- }
-
- ::arrow::MemoryPool* memory_pool() const { return pool_; }
-
- std::unique_ptr<InputStream> GetStream(RandomAccessSource* source, int64_t start,
- int64_t num_bytes) {
- std::unique_ptr<InputStream> stream;
- if (buffered_stream_enabled_) {
- stream.reset(
- new BufferedInputStream(pool_, buffer_size_, source, start, num_bytes));
- } else {
- stream.reset(new InMemoryInputStream(source, start, num_bytes));
- }
- return stream;
- }
-
- bool is_buffered_stream_enabled() const { return buffered_stream_enabled_; }
-
- void enable_buffered_stream() { buffered_stream_enabled_ = true; }
-
- void disable_buffered_stream() { buffered_stream_enabled_ = false; }
-
- void set_buffer_size(int64_t buf_size) { buffer_size_ = buf_size; }
-
- int64_t buffer_size() const { return buffer_size_; }
-
- private:
- ::arrow::MemoryPool* pool_;
- int64_t buffer_size_;
- bool buffered_stream_enabled_;
-};
-
-ReaderProperties PARQUET_EXPORT default_reader_properties();
-
-static constexpr int64_t DEFAULT_PAGE_SIZE = 1024 * 1024;
-static constexpr bool DEFAULT_IS_DICTIONARY_ENABLED = true;
-static constexpr int64_t DEFAULT_DICTIONARY_PAGE_SIZE_LIMIT = DEFAULT_PAGE_SIZE;
-static constexpr int64_t DEFAULT_WRITE_BATCH_SIZE = 1024;
-static constexpr int64_t DEFAULT_MAX_ROW_GROUP_LENGTH = 64 * 1024 * 1024;
-static constexpr bool DEFAULT_ARE_STATISTICS_ENABLED = true;
-static constexpr int64_t DEFAULT_MAX_STATISTICS_SIZE = 4096;
-static constexpr Encoding::type DEFAULT_ENCODING = Encoding::PLAIN;
-static constexpr ParquetVersion::type DEFAULT_WRITER_VERSION =
- ParquetVersion::PARQUET_1_0;
-static const char DEFAULT_CREATED_BY[] = CREATED_BY_VERSION;
-static constexpr Compression::type DEFAULT_COMPRESSION_TYPE = Compression::UNCOMPRESSED;
-
-class PARQUET_EXPORT ColumnProperties {
- public:
- ColumnProperties(Encoding::type encoding = DEFAULT_ENCODING,
- Compression::type codec = DEFAULT_COMPRESSION_TYPE,
- bool dictionary_enabled = DEFAULT_IS_DICTIONARY_ENABLED,
- bool statistics_enabled = DEFAULT_ARE_STATISTICS_ENABLED,
- size_t max_stats_size = DEFAULT_MAX_STATISTICS_SIZE)
- : encoding_(encoding),
- codec_(codec),
- dictionary_enabled_(dictionary_enabled),
- statistics_enabled_(statistics_enabled),
- max_stats_size_(max_stats_size) {}
-
- void set_encoding(Encoding::type encoding) { encoding_ = encoding; }
-
- void set_compression(Compression::type codec) { codec_ = codec; }
-
- void set_dictionary_enabled(bool dictionary_enabled) {
- dictionary_enabled_ = dictionary_enabled;
- }
-
- void set_statistics_enabled(bool statistics_enabled) {
- statistics_enabled_ = statistics_enabled;
- }
-
- void set_max_statistics_size(size_t max_stats_size) {
- max_stats_size_ = max_stats_size;
- }
-
- Encoding::type encoding() const { return encoding_; }
-
- Compression::type compression() const { return codec_; }
-
- bool dictionary_enabled() const { return dictionary_enabled_; }
-
- bool statistics_enabled() const { return statistics_enabled_; }
-
- size_t max_statistics_size() const { return max_stats_size_; }
-
- private:
- Encoding::type encoding_;
- Compression::type codec_;
- bool dictionary_enabled_;
- bool statistics_enabled_;
- size_t max_stats_size_;
-};
-
-class PARQUET_EXPORT WriterProperties {
- public:
- class Builder {
- public:
- Builder()
- : pool_(::arrow::default_memory_pool()),
- dictionary_pagesize_limit_(DEFAULT_DICTIONARY_PAGE_SIZE_LIMIT),
- write_batch_size_(DEFAULT_WRITE_BATCH_SIZE),
- max_row_group_length_(DEFAULT_MAX_ROW_GROUP_LENGTH),
- pagesize_(DEFAULT_PAGE_SIZE),
- version_(DEFAULT_WRITER_VERSION),
- created_by_(DEFAULT_CREATED_BY) {}
- virtual ~Builder() {}
-
- Builder* memory_pool(::arrow::MemoryPool* pool) {
- pool_ = pool;
- return this;
- }
-
- Builder* enable_dictionary() {
- default_column_properties_.set_dictionary_enabled(true);
- return this;
- }
-
- Builder* disable_dictionary() {
- default_column_properties_.set_dictionary_enabled(false);
- return this;
- }
-
- Builder* enable_dictionary(const std::string& path) {
- dictionary_enabled_[path] = true;
- return this;
- }
-
- Builder* enable_dictionary(const std::shared_ptr<schema::ColumnPath>& path) {
- return this->enable_dictionary(path->ToDotString());
- }
-
- Builder* disable_dictionary(const std::string& path) {
- dictionary_enabled_[path] = false;
- return this;
- }
-
- Builder* disable_dictionary(const std::shared_ptr<schema::ColumnPath>& path) {
- return this->disable_dictionary(path->ToDotString());
- }
-
- Builder* dictionary_pagesize_limit(int64_t dictionary_psize_limit) {
- dictionary_pagesize_limit_ = dictionary_psize_limit;
- return this;
- }
-
- Builder* write_batch_size(int64_t write_batch_size) {
- write_batch_size_ = write_batch_size;
- return this;
- }
-
- Builder* max_row_group_length(int64_t max_row_group_length) {
- max_row_group_length_ = max_row_group_length;
- return this;
- }
-
- Builder* data_pagesize(int64_t pg_size) {
- pagesize_ = pg_size;
- return this;
- }
-
- Builder* version(ParquetVersion::type version) {
- version_ = version;
- return this;
- }
-
- Builder* created_by(const std::string& created_by) {
- created_by_ = created_by;
- return this;
- }
-
- /**
- * Define the encoding that is used when we don't utilise dictionary encoding.
- *
- * This either apply if dictionary encoding is disabled or if we fallback
- * as the dictionary grew too large.
- */
- Builder* encoding(Encoding::type encoding_type) {
- if (encoding_type == Encoding::PLAIN_DICTIONARY ||
- encoding_type == Encoding::RLE_DICTIONARY) {
- throw ParquetException("Can't use dictionary encoding as fallback encoding");
- }
-
- default_column_properties_.set_encoding(encoding_type);
- return this;
- }
-
- /**
- * Define the encoding that is used when we don't utilise dictionary encoding.
- *
- * This either apply if dictionary encoding is disabled or if we fallback
- * as the dictionary grew too large.
- */
- Builder* encoding(const std::string& path, Encoding::type encoding_type) {
- if (encoding_type == Encoding::PLAIN_DICTIONARY ||
- encoding_type == Encoding::RLE_DICTIONARY) {
- throw ParquetException("Can't use dictionary encoding as fallback encoding");
- }
-
- encodings_[path] = encoding_type;
- return this;
- }
-
- /**
- * Define the encoding that is used when we don't utilise dictionary encoding.
- *
- * This either apply if dictionary encoding is disabled or if we fallback
- * as the dictionary grew too large.
- */
- Builder* encoding(const std::shared_ptr<schema::ColumnPath>& path,
- Encoding::type encoding_type) {
- return this->encoding(path->ToDotString(), encoding_type);
- }
-
- Builder* compression(Compression::type codec) {
- default_column_properties_.set_compression(codec);
- return this;
- }
-
- Builder* max_statistics_size(size_t max_stats_sz) {
- default_column_properties_.set_max_statistics_size(max_stats_sz);
- return this;
- }
-
- Builder* compression(const std::string& path, Compression::type codec) {
- codecs_[path] = codec;
- return this;
- }
-
- Builder* compression(const std::shared_ptr<schema::ColumnPath>& path,
- Compression::type codec) {
- return this->compression(path->ToDotString(), codec);
- }
-
- Builder* enable_statistics() {
- default_column_properties_.set_statistics_enabled(true);
- return this;
- }
-
- Builder* disable_statistics() {
- default_column_properties_.set_statistics_enabled(false);
- return this;
- }
-
- Builder* enable_statistics(const std::string& path) {
- statistics_enabled_[path] = true;
- return this;
- }
-
- Builder* enable_statistics(const std::shared_ptr<schema::ColumnPath>& path) {
- return this->enable_statistics(path->ToDotString());
- }
-
- Builder* disable_statistics(const std::string& path) {
- statistics_enabled_[path] = false;
- return this;
- }
-
- Builder* disable_statistics(const std::shared_ptr<schema::ColumnPath>& path) {
- return this->disable_statistics(path->ToDotString());
- }
-
- std::shared_ptr<WriterProperties> build() {
- std::unordered_map<std::string, ColumnProperties> column_properties;
- auto get = [&](const std::string& key) -> ColumnProperties& {
- auto it = column_properties.find(key);
- if (it == column_properties.end())
- return column_properties[key] = default_column_properties_;
- else
- return it->second;
- };
-
- for (const auto& item : encodings_) get(item.first).set_encoding(item.second);
- for (const auto& item : codecs_) get(item.first).set_compression(item.second);
- for (const auto& item : dictionary_enabled_)
- get(item.first).set_dictionary_enabled(item.second);
- for (const auto& item : statistics_enabled_)
- get(item.first).set_statistics_enabled(item.second);
-
- return std::shared_ptr<WriterProperties>(
- new WriterProperties(pool_, dictionary_pagesize_limit_, write_batch_size_,
- max_row_group_length_, pagesize_, version_, created_by_,
- default_column_properties_, column_properties));
- }
-
- private:
- ::arrow::MemoryPool* pool_;
- int64_t dictionary_pagesize_limit_;
- int64_t write_batch_size_;
- int64_t max_row_group_length_;
- int64_t pagesize_;
- ParquetVersion::type version_;
- std::string created_by_;
-
- // Settings used for each column unless overridden in any of the maps below
- ColumnProperties default_column_properties_;
- std::unordered_map<std::string, Encoding::type> encodings_;
- std::unordered_map<std::string, Compression::type> codecs_;
- std::unordered_map<std::string, bool> dictionary_enabled_;
- std::unordered_map<std::string, bool> statistics_enabled_;
- };
-
- inline ::arrow::MemoryPool* memory_pool() const { return pool_; }
-
- inline int64_t dictionary_pagesize_limit() const { return dictionary_pagesize_limit_; }
-
- inline int64_t write_batch_size() const { return write_batch_size_; }
-
- inline int64_t max_row_group_length() const { return max_row_group_length_; }
-
- inline int64_t data_pagesize() const { return pagesize_; }
-
- inline ParquetVersion::type version() const { return parquet_version_; }
-
- inline std::string created_by() const { return parquet_created_by_; }
-
- inline Encoding::type dictionary_index_encoding() const {
- if (parquet_version_ == ParquetVersion::PARQUET_1_0) {
- return Encoding::PLAIN_DICTIONARY;
- } else {
- return Encoding::RLE_DICTIONARY;
- }
- }
-
- inline Encoding::type dictionary_page_encoding() const {
- if (parquet_version_ == ParquetVersion::PARQUET_1_0) {
- return Encoding::PLAIN_DICTIONARY;
- } else {
- return Encoding::PLAIN;
- }
- }
-
- const ColumnProperties& column_properties(
- const std::shared_ptr<schema::ColumnPath>& path) const {
- auto it = column_properties_.find(path->ToDotString());
- if (it != column_properties_.end()) return it->second;
- return default_column_properties_;
- }
-
- Encoding::type encoding(const std::shared_ptr<schema::ColumnPath>& path) const {
- return column_properties(path).encoding();
- }
-
- Compression::type compression(const std::shared_ptr<schema::ColumnPath>& path) const {
- return column_properties(path).compression();
- }
-
- bool dictionary_enabled(const std::shared_ptr<schema::ColumnPath>& path) const {
- return column_properties(path).dictionary_enabled();
- }
-
- bool statistics_enabled(const std::shared_ptr<schema::ColumnPath>& path) const {
- return column_properties(path).statistics_enabled();
- }
-
- size_t max_statistics_size(const std::shared_ptr<schema::ColumnPath>& path) const {
- return column_properties(path).max_statistics_size();
- }
-
- private:
- explicit WriterProperties(
- ::arrow::MemoryPool* pool, int64_t dictionary_pagesize_limit,
- int64_t write_batch_size, int64_t max_row_group_length, int64_t pagesize,
- ParquetVersion::type version, const std::string& created_by,
- const ColumnProperties& default_column_properties,
- const std::unordered_map<std::string, ColumnProperties>& column_properties)
- : pool_(pool),
- dictionary_pagesize_limit_(dictionary_pagesize_limit),
- write_batch_size_(write_batch_size),
- max_row_group_length_(max_row_group_length),
- pagesize_(pagesize),
- parquet_version_(version),
- parquet_created_by_(created_by),
- default_column_properties_(default_column_properties),
- column_properties_(column_properties) {}
-
- ::arrow::MemoryPool* pool_;
- int64_t dictionary_pagesize_limit_;
- int64_t write_batch_size_;
- int64_t max_row_group_length_;
- int64_t pagesize_;
- ParquetVersion::type parquet_version_;
- std::string parquet_created_by_;
- ColumnProperties default_column_properties_;
- std::unordered_map<std::string, ColumnProperties> column_properties_;
-};
-
-std::shared_ptr<WriterProperties> PARQUET_EXPORT default_writer_properties();
-
-} // namespace parquet
-
-#endif // PARQUET_COLUMN_PROPERTIES_H
diff --git a/src/parquet/public-api-test.cc b/src/parquet/public-api-test.cc
deleted file mode 100644
index 958e970..0000000
--- a/src/parquet/public-api-test.cc
+++ /dev/null
@@ -1,49 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#include <gtest/gtest.h>
-
-#include "parquet/api/io.h"
-#include "parquet/api/reader.h"
-#include "parquet/api/schema.h"
-#include "parquet/api/writer.h"
-
-TEST(TestPublicAPI, DoesNotIncludeThrift) {
-#ifdef _THRIFT_THRIFT_H_
- FAIL() << "Thrift headers should not be in the public API";
-#endif
-}
-
-TEST(TestPublicAPI, DoesNotExportDCHECK) {
-#ifdef DCHECK
- FAIL() << "parquet/util/logging.h should not be transitively included";
-#endif
-}
-
-TEST(TestPublicAPI, DoesNotIncludeZlib) {
-#ifdef ZLIB_H
- FAIL() << "zlib.h should not be transitively included";
-#endif
-}
-
-PARQUET_NORETURN void ThrowsParquetException() {
- throw parquet::ParquetException("This function throws");
-}
-
-TEST(TestPublicAPI, CanThrowParquetException) {
- ASSERT_THROW(ThrowsParquetException(), parquet::ParquetException);
-}
diff --git a/src/parquet/reader-test.cc b/src/parquet/reader-test.cc
deleted file mode 100644
index d628f47..0000000
--- a/src/parquet/reader-test.cc
+++ /dev/null
@@ -1,325 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#include <fcntl.h>
-#include <gtest/gtest.h>
-#include <cstdint>
-#include <cstdlib>
-#include <iostream>
-#include <memory>
-#include <string>
-
-#include "arrow/io/file.h"
-
-#include "parquet/column_reader.h"
-#include "parquet/column_scanner.h"
-#include "parquet/file_reader.h"
-#include "parquet/printer.h"
-#include "parquet/util/memory.h"
-#include "parquet/util/test-common.h"
-
-using std::string;
-
-namespace parquet {
-
-using ReadableFile = ::arrow::io::ReadableFile;
-
-std::string alltypes_plain() {
- std::string dir_string(test::get_data_dir());
- std::stringstream ss;
- ss << dir_string << "/"
- << "alltypes_plain.parquet";
- return ss.str();
-}
-
-std::string nation_dict_truncated_data_page() {
- std::string dir_string(test::get_data_dir());
- std::stringstream ss;
- ss << dir_string << "/"
- << "nation.dict-malformed.parquet";
- return ss.str();
-}
-
-class TestAllTypesPlain : public ::testing::Test {
- public:
- void SetUp() { reader_ = ParquetFileReader::OpenFile(alltypes_plain()); }
-
- void TearDown() {}
-
- protected:
- std::unique_ptr<ParquetFileReader> reader_;
-};
-
-TEST_F(TestAllTypesPlain, NoopConstructDestruct) {}
-
-TEST_F(TestAllTypesPlain, TestBatchRead) {
- std::shared_ptr<RowGroupReader> group = reader_->RowGroup(0);
-
- // column 0, id
- std::shared_ptr<Int32Reader> col =
- std::dynamic_pointer_cast<Int32Reader>(group->Column(0));
-
- int16_t def_levels[4];
- int16_t rep_levels[4];
- int32_t values[4];
-
- // This file only has 8 rows
- ASSERT_EQ(8, reader_->metadata()->num_rows());
- // This file only has 1 row group
- ASSERT_EQ(1, reader_->metadata()->num_row_groups());
- // Size of the metadata is 730 bytes
- ASSERT_EQ(730, reader_->metadata()->size());
- // This row group must have 8 rows
- ASSERT_EQ(8, group->metadata()->num_rows());
-
- ASSERT_TRUE(col->HasNext());
- int64_t values_read;
- auto levels_read = col->ReadBatch(4, def_levels, rep_levels, values, &values_read);
- ASSERT_EQ(4, levels_read);
- ASSERT_EQ(4, values_read);
-
- // Now read past the end of the file
- ASSERT_TRUE(col->HasNext());
- levels_read = col->ReadBatch(5, def_levels, rep_levels, values, &values_read);
- ASSERT_EQ(4, levels_read);
- ASSERT_EQ(4, values_read);
-
- ASSERT_FALSE(col->HasNext());
-}
-
-TEST_F(TestAllTypesPlain, TestFlatScannerInt32) {
- std::shared_ptr<RowGroupReader> group = reader_->RowGroup(0);
-
- // column 0, id
- std::shared_ptr<Int32Scanner> scanner(new Int32Scanner(group->Column(0)));
- int32_t val;
- bool is_null;
- for (int i = 0; i < 8; ++i) {
- ASSERT_TRUE(scanner->HasNext());
- ASSERT_TRUE(scanner->NextValue(&val, &is_null));
- ASSERT_FALSE(is_null);
- }
- ASSERT_FALSE(scanner->HasNext());
- ASSERT_FALSE(scanner->NextValue(&val, &is_null));
-}
-
-TEST_F(TestAllTypesPlain, TestSetScannerBatchSize) {
- std::shared_ptr<RowGroupReader> group = reader_->RowGroup(0);
-
- // column 0, id
- std::shared_ptr<Int32Scanner> scanner(new Int32Scanner(group->Column(0)));
-
- ASSERT_EQ(128, scanner->batch_size());
- scanner->SetBatchSize(1024);
- ASSERT_EQ(1024, scanner->batch_size());
-}
-
-TEST_F(TestAllTypesPlain, DebugPrintWorks) {
- std::stringstream ss;
-
- std::list<int> columns;
- ParquetFilePrinter printer(reader_.get());
- printer.DebugPrint(ss, columns);
-
- std::string result = ss.str();
- ASSERT_GT(result.size(), 0);
-}
-
-TEST_F(TestAllTypesPlain, ColumnSelection) {
- std::stringstream ss;
-
- std::list<int> columns;
- columns.push_back(5);
- columns.push_back(0);
- columns.push_back(10);
- ParquetFilePrinter printer(reader_.get());
- printer.DebugPrint(ss, columns);
-
- std::string result = ss.str();
- ASSERT_GT(result.size(), 0);
-}
-
-TEST_F(TestAllTypesPlain, ColumnSelectionOutOfRange) {
- std::stringstream ss;
-
- std::list<int> columns;
- columns.push_back(100);
- ParquetFilePrinter printer1(reader_.get());
- ASSERT_THROW(printer1.DebugPrint(ss, columns), ParquetException);
-
- columns.clear();
- columns.push_back(-1);
- ParquetFilePrinter printer2(reader_.get());
- ASSERT_THROW(printer2.DebugPrint(ss, columns), ParquetException);
-}
-
-class TestLocalFile : public ::testing::Test {
- public:
- void SetUp() {
- std::string dir_string(test::get_data_dir());
-
- std::stringstream ss;
- ss << dir_string << "/"
- << "alltypes_plain.parquet";
-
- PARQUET_THROW_NOT_OK(ReadableFile::Open(ss.str(), &handle));
- fileno = handle->file_descriptor();
- }
-
- void TearDown() {}
-
- protected:
- int fileno;
- std::shared_ptr<::arrow::io::ReadableFile> handle;
-};
-
-class HelperFileClosed : public ArrowInputFile {
- public:
- explicit HelperFileClosed(
- const std::shared_ptr<::arrow::io::ReadableFileInterface>& file, bool* close_called)
- : ArrowInputFile(file), close_called_(close_called) {}
-
- void Close() override { *close_called_ = true; }
-
- private:
- bool* close_called_;
-};
-
-TEST_F(TestLocalFile, FileClosedOnDestruction) {
- bool close_called = false;
- {
- auto contents = ParquetFileReader::Contents::Open(
- std::unique_ptr<RandomAccessSource>(new HelperFileClosed(handle, &close_called)));
- std::unique_ptr<ParquetFileReader> result(new ParquetFileReader());
- result->Open(std::move(contents));
- }
- ASSERT_TRUE(close_called);
-}
-
-TEST_F(TestLocalFile, OpenWithMetadata) {
- // PARQUET-808
- std::stringstream ss;
- std::shared_ptr<FileMetaData> metadata = ReadMetaData(handle);
-
- auto reader = ParquetFileReader::Open(handle, default_reader_properties(), metadata);
-
- // Compare pointers
- ASSERT_EQ(metadata.get(), reader->metadata().get());
-
- std::list<int> columns;
- ParquetFilePrinter printer(reader.get());
- printer.DebugPrint(ss, columns, true);
-
- // Make sure OpenFile passes on the external metadata, too
- auto reader2 = ParquetFileReader::OpenFile(alltypes_plain(), false,
- default_reader_properties(), metadata);
-
- // Compare pointers
- ASSERT_EQ(metadata.get(), reader2->metadata().get());
-}
-
-TEST(TestFileReaderAdHoc, NationDictTruncatedDataPage) {
- // PARQUET-816. Some files generated by older Parquet implementations may
- // contain malformed data page metadata, and we can successfully decode them
- // if we optimistically proceed to decoding, even if there is not enough data
- // available in the stream. Before, we had quite aggressive checking of
- // stream reads, which are not found e.g. in Impala's Parquet implementation
- auto reader = ParquetFileReader::OpenFile(nation_dict_truncated_data_page(), false);
- std::stringstream ss;
-
- // empty list means print all
- std::list<int> columns;
- ParquetFilePrinter printer1(reader.get());
- printer1.DebugPrint(ss, columns, true);
-
- reader = ParquetFileReader::OpenFile(nation_dict_truncated_data_page(), true);
- std::stringstream ss2;
- ParquetFilePrinter printer2(reader.get());
- printer2.DebugPrint(ss2, columns, true);
-
- // The memory-mapped reads runs over the end of the column chunk and succeeds
- // by accident
- ASSERT_EQ(ss2.str(), ss.str());
-}
-
-TEST(TestJSONWithLocalFile, JSONOutput) {
- std::string jsonOutput = R"###({
- "FileName": "alltypes_plain.parquet",
- "Version": "0",
- "CreatedBy": "impala version 1.3.0-INTERNAL (build 8a48ddb1eff84592b3fc06bc6f51ec120e1fffc9)",
- "TotalRows": "8",
- "NumberOfRowGroups": "1",
- "NumberOfRealColumns": "11",
- "NumberOfColumns": "11",
- "Columns": [
- { "Id": "0", "Name": "id", "PhysicalType": "INT32", "LogicalType": "NONE" },
- { "Id": "1", "Name": "bool_col", "PhysicalType": "BOOLEAN", "LogicalType": "NONE" },
- { "Id": "2", "Name": "tinyint_col", "PhysicalType": "INT32", "LogicalType": "NONE" },
- { "Id": "3", "Name": "smallint_col", "PhysicalType": "INT32", "LogicalType": "NONE" },
- { "Id": "4", "Name": "int_col", "PhysicalType": "INT32", "LogicalType": "NONE" },
- { "Id": "5", "Name": "bigint_col", "PhysicalType": "INT64", "LogicalType": "NONE" },
- { "Id": "6", "Name": "float_col", "PhysicalType": "FLOAT", "LogicalType": "NONE" },
- { "Id": "7", "Name": "double_col", "PhysicalType": "DOUBLE", "LogicalType": "NONE" },
- { "Id": "8", "Name": "date_string_col", "PhysicalType": "BYTE_ARRAY", "LogicalType": "NONE" },
- { "Id": "9", "Name": "string_col", "PhysicalType": "BYTE_ARRAY", "LogicalType": "NONE" },
- { "Id": "10", "Name": "timestamp_col", "PhysicalType": "INT96", "LogicalType": "NONE" }
- ],
- "RowGroups": [
- {
- "Id": "0", "TotalBytes": "671", "Rows": "8",
- "ColumnChunks": [
- {"Id": "0", "Values": "8", "StatsSet": "False",
- "Compression": "UNCOMPRESSED", "Encodings": "RLE PLAIN_DICTIONARY PLAIN ", "UncompressedSize": "73", "CompressedSize": "73" },
- {"Id": "1", "Values": "8", "StatsSet": "False",
- "Compression": "UNCOMPRESSED", "Encodings": "RLE PLAIN_DICTIONARY PLAIN ", "UncompressedSize": "24", "CompressedSize": "24" },
- {"Id": "2", "Values": "8", "StatsSet": "False",
- "Compression": "UNCOMPRESSED", "Encodings": "RLE PLAIN_DICTIONARY PLAIN ", "UncompressedSize": "47", "CompressedSize": "47" },
- {"Id": "3", "Values": "8", "StatsSet": "False",
- "Compression": "UNCOMPRESSED", "Encodings": "RLE PLAIN_DICTIONARY PLAIN ", "UncompressedSize": "47", "CompressedSize": "47" },
- {"Id": "4", "Values": "8", "StatsSet": "False",
- "Compression": "UNCOMPRESSED", "Encodings": "RLE PLAIN_DICTIONARY PLAIN ", "UncompressedSize": "47", "CompressedSize": "47" },
- {"Id": "5", "Values": "8", "StatsSet": "False",
- "Compression": "UNCOMPRESSED", "Encodings": "RLE PLAIN_DICTIONARY PLAIN ", "UncompressedSize": "55", "CompressedSize": "55" },
- {"Id": "6", "Values": "8", "StatsSet": "False",
- "Compression": "UNCOMPRESSED", "Encodings": "RLE PLAIN_DICTIONARY PLAIN ", "UncompressedSize": "47", "CompressedSize": "47" },
- {"Id": "7", "Values": "8", "StatsSet": "False",
- "Compression": "UNCOMPRESSED", "Encodings": "RLE PLAIN_DICTIONARY PLAIN ", "UncompressedSize": "55", "CompressedSize": "55" },
- {"Id": "8", "Values": "8", "StatsSet": "False",
- "Compression": "UNCOMPRESSED", "Encodings": "RLE PLAIN_DICTIONARY PLAIN ", "UncompressedSize": "88", "CompressedSize": "88" },
- {"Id": "9", "Values": "8", "StatsSet": "False",
- "Compression": "UNCOMPRESSED", "Encodings": "RLE PLAIN_DICTIONARY PLAIN ", "UncompressedSize": "49", "CompressedSize": "49" },
- {"Id": "10", "Values": "8", "StatsSet": "False",
- "Compression": "UNCOMPRESSED", "Encodings": "RLE PLAIN_DICTIONARY PLAIN ", "UncompressedSize": "139", "CompressedSize": "139" }
- ]
- }
- ]
-}
-)###";
-
- std::stringstream ss;
- // empty list means print all
- std::list<int> columns;
-
- auto reader =
- ParquetFileReader::OpenFile(alltypes_plain(), false, default_reader_properties());
- ParquetFilePrinter printer(reader.get());
- printer.JSONPrint(ss, columns, "alltypes_plain.parquet");
-
- ASSERT_EQ(jsonOutput, ss.str());
-}
-
-} // namespace parquet
diff --git a/src/parquet/schema-internal.h b/src/parquet/schema-internal.h
deleted file mode 100644
index 53472ab..0000000
--- a/src/parquet/schema-internal.h
+++ /dev/null
@@ -1,83 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-// This module contains the logical parquet-cpp types (independent of Thrift
-// structures), schema nodes, and related type tools
-
-#ifndef PARQUET_SCHEMA_INTERNAL_H
-#define PARQUET_SCHEMA_INTERNAL_H
-
-#include <cstdint>
-#include <memory>
-#include <vector>
-
-#include "parquet/parquet_types.h"
-#include "parquet/schema.h"
-#include "parquet/types.h"
-#include "parquet/util/macros.h"
-#include "parquet/util/visibility.h"
-
-namespace parquet {
-namespace schema {
-
-// ----------------------------------------------------------------------
-// Conversion from Parquet Thrift metadata
-
-std::shared_ptr<SchemaDescriptor> FromParquet(
- const std::vector<format::SchemaElement>& schema);
-
-class FlatSchemaConverter {
- public:
- FlatSchemaConverter(const format::SchemaElement* elements, int length)
- : elements_(elements), length_(length), pos_(0), current_id_(0) {}
-
- std::unique_ptr<Node> Convert();
-
- private:
- const format::SchemaElement* elements_;
- int length_;
- int pos_;
- int current_id_;
-
- int next_id() { return current_id_++; }
-
- const format::SchemaElement& Next();
-
- std::unique_ptr<Node> NextNode();
-};
-
-// ----------------------------------------------------------------------
-// Conversion to Parquet Thrift metadata
-
-void ToParquet(const GroupNode* schema, std::vector<format::SchemaElement>* out);
-
-// Converts nested parquet schema back to a flat vector of Thrift structs
-class SchemaFlattener {
- public:
- SchemaFlattener(const GroupNode* schema, std::vector<format::SchemaElement>* out);
-
- void Flatten();
-
- private:
- const GroupNode* root_;
- std::vector<format::SchemaElement>* elements_;
-};
-
-} // namespace schema
-} // namespace parquet
-
-#endif // PARQUET_SCHEMA_INTERNAL_H
diff --git a/src/parquet/schema-test.cc b/src/parquet/schema-test.cc
deleted file mode 100644
index a734a92..0000000
--- a/src/parquet/schema-test.cc
+++ /dev/null
@@ -1,761 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#include <gtest/gtest.h>
-
-#include <cstdlib>
-#include <iosfwd>
-#include <memory>
-#include <string>
-#include <vector>
-
-#include "parquet/exception.h"
-#include "parquet/parquet_types.h"
-#include "parquet/schema-internal.h"
-#include "parquet/schema.h"
-#include "parquet/types.h"
-
-using std::string;
-using std::vector;
-
-namespace parquet {
-
-using format::ConvertedType;
-using format::FieldRepetitionType;
-using format::SchemaElement;
-
-namespace schema {
-
-static inline SchemaElement NewPrimitive(const std::string& name,
- FieldRepetitionType::type repetition,
- format::Type::type type, int id = 0) {
- SchemaElement result;
- result.__set_name(name);
- result.__set_repetition_type(repetition);
- result.__set_type(type);
-
- return result;
-}
-
-static inline SchemaElement NewGroup(const std::string& name,
- FieldRepetitionType::type repetition,
- int num_children, int id = 0) {
- SchemaElement result;
- result.__set_name(name);
- result.__set_repetition_type(repetition);
- result.__set_num_children(num_children);
-
- return result;
-}
-
-// ----------------------------------------------------------------------
-// ColumnPath
-
-TEST(TestColumnPath, TestAttrs) {
- ColumnPath path(std::vector<std::string>({"toplevel", "leaf"}));
-
- ASSERT_EQ(path.ToDotString(), "toplevel.leaf");
-
- std::shared_ptr<ColumnPath> path_ptr = ColumnPath::FromDotString("toplevel.leaf");
- ASSERT_EQ(path_ptr->ToDotString(), "toplevel.leaf");
-
- std::shared_ptr<ColumnPath> extended = path_ptr->extend("anotherlevel");
- ASSERT_EQ(extended->ToDotString(), "toplevel.leaf.anotherlevel");
-}
-
-// ----------------------------------------------------------------------
-// Primitive node
-
-class TestPrimitiveNode : public ::testing::Test {
- public:
- void SetUp() {
- name_ = "name";
- id_ = 5;
- }
-
- void Convert(const format::SchemaElement* element) {
- node_ = PrimitiveNode::FromParquet(element, id_);
- ASSERT_TRUE(node_->is_primitive());
- prim_node_ = static_cast<const PrimitiveNode*>(node_.get());
- }
-
- protected:
- std::string name_;
- const PrimitiveNode* prim_node_;
-
- int id_;
- std::unique_ptr<Node> node_;
-};
-
-TEST_F(TestPrimitiveNode, Attrs) {
- PrimitiveNode node1("foo", Repetition::REPEATED, Type::INT32);
-
- PrimitiveNode node2("bar", Repetition::OPTIONAL, Type::BYTE_ARRAY, LogicalType::UTF8);
-
- ASSERT_EQ("foo", node1.name());
-
- ASSERT_TRUE(node1.is_primitive());
- ASSERT_FALSE(node1.is_group());
-
- ASSERT_EQ(Repetition::REPEATED, node1.repetition());
- ASSERT_EQ(Repetition::OPTIONAL, node2.repetition());
-
- ASSERT_EQ(Node::PRIMITIVE, node1.node_type());
-
- ASSERT_EQ(Type::INT32, node1.physical_type());
- ASSERT_EQ(Type::BYTE_ARRAY, node2.physical_type());
-
- // logical types
- ASSERT_EQ(LogicalType::NONE, node1.logical_type());
- ASSERT_EQ(LogicalType::UTF8, node2.logical_type());
-
- // repetition
- PrimitiveNode node3("foo", Repetition::REPEATED, Type::INT32);
- PrimitiveNode node4("foo", Repetition::REQUIRED, Type::INT32);
- PrimitiveNode node5("foo", Repetition::OPTIONAL, Type::INT32);
-
- ASSERT_TRUE(node3.is_repeated());
- ASSERT_FALSE(node3.is_optional());
-
- ASSERT_TRUE(node4.is_required());
-
- ASSERT_TRUE(node5.is_optional());
- ASSERT_FALSE(node5.is_required());
-}
-
-TEST_F(TestPrimitiveNode, FromParquet) {
- SchemaElement elt =
- NewPrimitive(name_, FieldRepetitionType::OPTIONAL, format::Type::INT32, 0);
- ASSERT_NO_FATAL_FAILURE(Convert(&elt));
- ASSERT_EQ(name_, prim_node_->name());
- ASSERT_EQ(id_, prim_node_->id());
- ASSERT_EQ(Repetition::OPTIONAL, prim_node_->repetition());
- ASSERT_EQ(Type::INT32, prim_node_->physical_type());
- ASSERT_EQ(LogicalType::NONE, prim_node_->logical_type());
-
- // Test a logical type
- elt = NewPrimitive(name_, FieldRepetitionType::REQUIRED, format::Type::BYTE_ARRAY, 0);
- elt.__set_converted_type(ConvertedType::UTF8);
-
- ASSERT_NO_FATAL_FAILURE(Convert(&elt));
- ASSERT_EQ(Repetition::REQUIRED, prim_node_->repetition());
- ASSERT_EQ(Type::BYTE_ARRAY, prim_node_->physical_type());
- ASSERT_EQ(LogicalType::UTF8, prim_node_->logical_type());
-
- // FIXED_LEN_BYTE_ARRAY
- elt = NewPrimitive(name_, FieldRepetitionType::OPTIONAL,
- format::Type::FIXED_LEN_BYTE_ARRAY, 0);
- elt.__set_type_length(16);
-
- ASSERT_NO_FATAL_FAILURE(Convert(&elt));
- ASSERT_EQ(name_, prim_node_->name());
- ASSERT_EQ(id_, prim_node_->id());
- ASSERT_EQ(Repetition::OPTIONAL, prim_node_->repetition());
- ASSERT_EQ(Type::FIXED_LEN_BYTE_ARRAY, prim_node_->physical_type());
- ASSERT_EQ(16, prim_node_->type_length());
-
- // ConvertedType::Decimal
- elt = NewPrimitive(name_, FieldRepetitionType::OPTIONAL,
- format::Type::FIXED_LEN_BYTE_ARRAY, 0);
- elt.__set_converted_type(ConvertedType::DECIMAL);
- elt.__set_type_length(6);
- elt.__set_scale(2);
- elt.__set_precision(12);
-
- ASSERT_NO_FATAL_FAILURE(Convert(&elt));
- ASSERT_EQ(Type::FIXED_LEN_BYTE_ARRAY, prim_node_->physical_type());
- ASSERT_EQ(LogicalType::DECIMAL, prim_node_->logical_type());
- ASSERT_EQ(6, prim_node_->type_length());
- ASSERT_EQ(2, prim_node_->decimal_metadata().scale);
- ASSERT_EQ(12, prim_node_->decimal_metadata().precision);
-}
-
-TEST_F(TestPrimitiveNode, Equals) {
- PrimitiveNode node1("foo", Repetition::REQUIRED, Type::INT32);
- PrimitiveNode node2("foo", Repetition::REQUIRED, Type::INT64);
- PrimitiveNode node3("bar", Repetition::REQUIRED, Type::INT32);
- PrimitiveNode node4("foo", Repetition::OPTIONAL, Type::INT32);
- PrimitiveNode node5("foo", Repetition::REQUIRED, Type::INT32);
-
- ASSERT_TRUE(node1.Equals(&node1));
- ASSERT_FALSE(node1.Equals(&node2));
- ASSERT_FALSE(node1.Equals(&node3));
- ASSERT_FALSE(node1.Equals(&node4));
- ASSERT_TRUE(node1.Equals(&node5));
-
- PrimitiveNode flba1("foo", Repetition::REQUIRED, Type::FIXED_LEN_BYTE_ARRAY,
- LogicalType::DECIMAL, 12, 4, 2);
-
- PrimitiveNode flba2("foo", Repetition::REQUIRED, Type::FIXED_LEN_BYTE_ARRAY,
- LogicalType::DECIMAL, 1, 4, 2);
- flba2.SetTypeLength(12);
-
- PrimitiveNode flba3("foo", Repetition::REQUIRED, Type::FIXED_LEN_BYTE_ARRAY,
- LogicalType::DECIMAL, 1, 4, 2);
- flba3.SetTypeLength(16);
-
- PrimitiveNode flba4("foo", Repetition::REQUIRED, Type::FIXED_LEN_BYTE_ARRAY,
- LogicalType::DECIMAL, 12, 4, 0);
-
- PrimitiveNode flba5("foo", Repetition::REQUIRED, Type::FIXED_LEN_BYTE_ARRAY,
- LogicalType::NONE, 12, 4, 0);
-
- ASSERT_TRUE(flba1.Equals(&flba2));
- ASSERT_FALSE(flba1.Equals(&flba3));
- ASSERT_FALSE(flba1.Equals(&flba4));
- ASSERT_FALSE(flba1.Equals(&flba5));
-}
-
-TEST_F(TestPrimitiveNode, PhysicalLogicalMapping) {
- ASSERT_NO_THROW(
- PrimitiveNode::Make("foo", Repetition::REQUIRED, Type::INT32, LogicalType::INT_32));
- ASSERT_NO_THROW(PrimitiveNode::Make("foo", Repetition::REQUIRED, Type::BYTE_ARRAY,
- LogicalType::JSON));
- ASSERT_THROW(
- PrimitiveNode::Make("foo", Repetition::REQUIRED, Type::INT32, LogicalType::JSON),
- ParquetException);
- ASSERT_NO_THROW(PrimitiveNode::Make("foo", Repetition::REQUIRED, Type::INT64,
- LogicalType::TIMESTAMP_MILLIS));
- ASSERT_THROW(
- PrimitiveNode::Make("foo", Repetition::REQUIRED, Type::INT32, LogicalType::INT_64),
- ParquetException);
- ASSERT_THROW(PrimitiveNode::Make("foo", Repetition::REQUIRED, Type::BYTE_ARRAY,
- LogicalType::INT_8),
- ParquetException);
- ASSERT_THROW(PrimitiveNode::Make("foo", Repetition::REQUIRED, Type::BYTE_ARRAY,
- LogicalType::INTERVAL),
- ParquetException);
- ASSERT_THROW(PrimitiveNode::Make("foo", Repetition::REQUIRED,
- Type::FIXED_LEN_BYTE_ARRAY, LogicalType::ENUM),
- ParquetException);
- ASSERT_NO_THROW(PrimitiveNode::Make("foo", Repetition::REQUIRED, Type::BYTE_ARRAY,
- LogicalType::ENUM));
- ASSERT_THROW(
- PrimitiveNode::Make("foo", Repetition::REQUIRED, Type::FIXED_LEN_BYTE_ARRAY,
- LogicalType::DECIMAL, 0, 2, 4),
- ParquetException);
- ASSERT_THROW(PrimitiveNode::Make("foo", Repetition::REQUIRED, Type::FLOAT,
- LogicalType::DECIMAL, 0, 2, 4),
- ParquetException);
- ASSERT_THROW(
- PrimitiveNode::Make("foo", Repetition::REQUIRED, Type::FIXED_LEN_BYTE_ARRAY,
- LogicalType::DECIMAL, 0, 4, 0),
- ParquetException);
- ASSERT_THROW(
- PrimitiveNode::Make("foo", Repetition::REQUIRED, Type::FIXED_LEN_BYTE_ARRAY,
- LogicalType::DECIMAL, 10, 0, 4),
- ParquetException);
- ASSERT_THROW(
- PrimitiveNode::Make("foo", Repetition::REQUIRED, Type::FIXED_LEN_BYTE_ARRAY,
- LogicalType::DECIMAL, 10, 4, -1),
- ParquetException);
- ASSERT_THROW(
- PrimitiveNode::Make("foo", Repetition::REQUIRED, Type::FIXED_LEN_BYTE_ARRAY,
- LogicalType::DECIMAL, 10, 2, 4),
- ParquetException);
- ASSERT_NO_THROW(PrimitiveNode::Make("foo", Repetition::REQUIRED,
- Type::FIXED_LEN_BYTE_ARRAY, LogicalType::DECIMAL,
- 10, 6, 4));
- ASSERT_NO_THROW(PrimitiveNode::Make("foo", Repetition::REQUIRED,
- Type::FIXED_LEN_BYTE_ARRAY, LogicalType::INTERVAL,
- 12));
- ASSERT_THROW(PrimitiveNode::Make("foo", Repetition::REQUIRED,
- Type::FIXED_LEN_BYTE_ARRAY, LogicalType::INTERVAL, 10),
- ParquetException);
-}
-
-// ----------------------------------------------------------------------
-// Group node
-
-class TestGroupNode : public ::testing::Test {
- public:
- NodeVector Fields1() {
- NodeVector fields;
-
- fields.push_back(Int32("one", Repetition::REQUIRED));
- fields.push_back(Int64("two"));
- fields.push_back(Double("three"));
-
- return fields;
- }
-
- NodeVector Fields2() {
- // Fields with a duplicate name
- NodeVector fields;
-
- fields.push_back(Int32("duplicate", Repetition::REQUIRED));
- fields.push_back(Int64("unique"));
- fields.push_back(Double("duplicate"));
-
- return fields;
- }
-};
-
-TEST_F(TestGroupNode, Attrs) {
- NodeVector fields = Fields1();
-
- GroupNode node1("foo", Repetition::REPEATED, fields);
- GroupNode node2("bar", Repetition::OPTIONAL, fields, LogicalType::LIST);
-
- ASSERT_EQ("foo", node1.name());
-
- ASSERT_TRUE(node1.is_group());
- ASSERT_FALSE(node1.is_primitive());
-
- ASSERT_EQ(fields.size(), node1.field_count());
-
- ASSERT_TRUE(node1.is_repeated());
- ASSERT_TRUE(node2.is_optional());
-
- ASSERT_EQ(Repetition::REPEATED, node1.repetition());
- ASSERT_EQ(Repetition::OPTIONAL, node2.repetition());
-
- ASSERT_EQ(Node::GROUP, node1.node_type());
-
- // logical types
- ASSERT_EQ(LogicalType::NONE, node1.logical_type());
- ASSERT_EQ(LogicalType::LIST, node2.logical_type());
-}
-
-TEST_F(TestGroupNode, Equals) {
- NodeVector f1 = Fields1();
- NodeVector f2 = Fields1();
-
- GroupNode group1("group", Repetition::REPEATED, f1);
- GroupNode group2("group", Repetition::REPEATED, f2);
- GroupNode group3("group2", Repetition::REPEATED, f2);
-
- // This is copied in the GroupNode ctor, so this is okay
- f2.push_back(Float("four", Repetition::OPTIONAL));
- GroupNode group4("group", Repetition::REPEATED, f2);
- GroupNode group5("group", Repetition::REPEATED, Fields1());
-
- ASSERT_TRUE(group1.Equals(&group1));
- ASSERT_TRUE(group1.Equals(&group2));
- ASSERT_FALSE(group1.Equals(&group3));
-
- ASSERT_FALSE(group1.Equals(&group4));
- ASSERT_FALSE(group5.Equals(&group4));
-}
-
-TEST_F(TestGroupNode, FieldIndex) {
- NodeVector fields = Fields1();
- GroupNode group("group", Repetition::REQUIRED, fields);
- for (size_t i = 0; i < fields.size(); i++) {
- auto field = group.field(static_cast<int>(i));
- ASSERT_EQ(i, group.FieldIndex(*field));
- }
-
- // Test a non field node
- auto non_field_alien = Int32("alien", Repetition::REQUIRED); // other name
- auto non_field_familiar = Int32("one", Repetition::REPEATED); // other node
- ASSERT_TRUE(group.FieldIndex(*non_field_alien) < 0);
- ASSERT_TRUE(group.FieldIndex(*non_field_familiar) < 0);
-}
-
-TEST_F(TestGroupNode, FieldIndexDuplicateName) {
- NodeVector fields = Fields2();
- GroupNode group("group", Repetition::REQUIRED, fields);
- for (size_t i = 0; i < fields.size(); i++) {
- auto field = group.field(static_cast<int>(i));
- ASSERT_EQ(i, group.FieldIndex(*field));
- }
-}
-
-// ----------------------------------------------------------------------
-// Test convert group
-
-class TestSchemaConverter : public ::testing::Test {
- public:
- void setUp() { name_ = "parquet_schema"; }
-
- void Convert(const parquet::format::SchemaElement* elements, int length) {
- FlatSchemaConverter converter(elements, length);
- node_ = converter.Convert();
- ASSERT_TRUE(node_->is_group());
- group_ = static_cast<const GroupNode*>(node_.get());
- }
-
- protected:
- std::string name_;
- const GroupNode* group_;
- std::unique_ptr<Node> node_;
-};
-
-bool check_for_parent_consistency(const GroupNode* node) {
- // Each node should have the group as parent
- for (int i = 0; i < node->field_count(); i++) {
- const NodePtr& field = node->field(i);
- if (field->parent() != node) {
- return false;
- }
- if (field->is_group()) {
- const GroupNode* group = static_cast<GroupNode*>(field.get());
- if (!check_for_parent_consistency(group)) {
- return false;
- }
- }
- }
- return true;
-}
-
-TEST_F(TestSchemaConverter, NestedExample) {
- SchemaElement elt;
- std::vector<SchemaElement> elements;
- elements.push_back(NewGroup(name_, FieldRepetitionType::REPEATED, 2, 0));
-
- // A primitive one
- elements.push_back(
- NewPrimitive("a", FieldRepetitionType::REQUIRED, format::Type::INT32, 1));
-
- // A group
- elements.push_back(NewGroup("bag", FieldRepetitionType::OPTIONAL, 1, 2));
-
- // 3-level list encoding, by hand
- elt = NewGroup("b", FieldRepetitionType::REPEATED, 1, 3);
- elt.__set_converted_type(ConvertedType::LIST);
- elements.push_back(elt);
- elements.push_back(
- NewPrimitive("item", FieldRepetitionType::OPTIONAL, format::Type::INT64, 4));
-
- ASSERT_NO_FATAL_FAILURE(Convert(&elements[0], static_cast<int>(elements.size())));
-
- // Construct the expected schema
- NodeVector fields;
- fields.push_back(Int32("a", Repetition::REQUIRED));
-
- // 3-level list encoding
- NodePtr item = Int64("item");
- NodePtr list(GroupNode::Make("b", Repetition::REPEATED, {item}, LogicalType::LIST));
- NodePtr bag(GroupNode::Make("bag", Repetition::OPTIONAL, {list}));
- fields.push_back(bag);
-
- NodePtr schema = GroupNode::Make(name_, Repetition::REPEATED, fields);
-
- ASSERT_TRUE(schema->Equals(group_));
-
- // Check that the parent relationship in each node is consitent
- ASSERT_EQ(group_->parent(), nullptr);
- ASSERT_TRUE(check_for_parent_consistency(group_));
-}
-
-TEST_F(TestSchemaConverter, InvalidRoot) {
- // According to the Parquet specification, the first element in the
- // list<SchemaElement> is a group whose children (and their descendants)
- // contain all of the rest of the flattened schema elements. If the first
- // element is not a group, it is a malformed Parquet file.
-
- SchemaElement elements[2];
- elements[0] =
- NewPrimitive("not-a-group", FieldRepetitionType::REQUIRED, format::Type::INT32, 0);
- ASSERT_THROW(Convert(elements, 2), ParquetException);
-
- // While the Parquet spec indicates that the root group should have REPEATED
- // repetition type, some implementations may return REQUIRED or OPTIONAL
- // groups as the first element. These tests check that this is okay as a
- // practicality matter.
- elements[0] = NewGroup("not-repeated", FieldRepetitionType::REQUIRED, 1, 0);
- elements[1] = NewPrimitive("a", FieldRepetitionType::REQUIRED, format::Type::INT32, 1);
- ASSERT_NO_FATAL_FAILURE(Convert(elements, 2));
-
- elements[0] = NewGroup("not-repeated", FieldRepetitionType::OPTIONAL, 1, 0);
- ASSERT_NO_FATAL_FAILURE(Convert(elements, 2));
-}
-
-TEST_F(TestSchemaConverter, NotEnoughChildren) {
- // Throw a ParquetException, but don't core dump or anything
- SchemaElement elt;
- std::vector<SchemaElement> elements;
- elements.push_back(NewGroup(name_, FieldRepetitionType::REPEATED, 2, 0));
- ASSERT_THROW(Convert(&elements[0], 1), ParquetException);
-}
-
-// ----------------------------------------------------------------------
-// Schema tree flatten / unflatten
-
-class TestSchemaFlatten : public ::testing::Test {
- public:
- void setUp() { name_ = "parquet_schema"; }
-
- void Flatten(const GroupNode* schema) { ToParquet(schema, &elements_); }
-
- protected:
- std::string name_;
- std::vector<format::SchemaElement> elements_;
-};
-
-TEST_F(TestSchemaFlatten, DecimalMetadata) {
- // Checks that DecimalMetadata is only set for DecimalTypes
- NodePtr node = PrimitiveNode::Make("decimal", Repetition::REQUIRED, Type::INT64,
- LogicalType::DECIMAL, -1, 8, 4);
- NodePtr group =
- GroupNode::Make("group", Repetition::REPEATED, {node}, LogicalType::LIST);
- Flatten(reinterpret_cast<GroupNode*>(group.get()));
- ASSERT_EQ("decimal", elements_[1].name);
- ASSERT_TRUE(elements_[1].__isset.precision);
- ASSERT_TRUE(elements_[1].__isset.scale);
-
- elements_.clear();
- // Not for integers with no logical type
- group =
- GroupNode::Make("group", Repetition::REPEATED, {Int64("int64")}, LogicalType::LIST);
- Flatten(reinterpret_cast<GroupNode*>(group.get()));
- ASSERT_EQ("int64", elements_[1].name);
- ASSERT_FALSE(elements_[0].__isset.precision);
- ASSERT_FALSE(elements_[0].__isset.scale);
-}
-
-TEST_F(TestSchemaFlatten, NestedExample) {
- SchemaElement elt;
- std::vector<SchemaElement> elements;
- elements.push_back(NewGroup(name_, FieldRepetitionType::REPEATED, 2, 0));
-
- // A primitive one
- elements.push_back(
- NewPrimitive("a", FieldRepetitionType::REQUIRED, format::Type::INT32, 1));
-
- // A group
- elements.push_back(NewGroup("bag", FieldRepetitionType::OPTIONAL, 1, 2));
-
- // 3-level list encoding, by hand
- elt = NewGroup("b", FieldRepetitionType::REPEATED, 1, 3);
- elt.__set_converted_type(ConvertedType::LIST);
- elements.push_back(elt);
- elements.push_back(
- NewPrimitive("item", FieldRepetitionType::OPTIONAL, format::Type::INT64, 4));
-
- // Construct the schema
- NodeVector fields;
- fields.push_back(Int32("a", Repetition::REQUIRED));
-
- // 3-level list encoding
- NodePtr item = Int64("item");
- NodePtr list(GroupNode::Make("b", Repetition::REPEATED, {item}, LogicalType::LIST));
- NodePtr bag(GroupNode::Make("bag", Repetition::OPTIONAL, {list}));
- fields.push_back(bag);
-
- NodePtr schema = GroupNode::Make(name_, Repetition::REPEATED, fields);
-
- Flatten(static_cast<GroupNode*>(schema.get()));
- ASSERT_EQ(elements_.size(), elements.size());
- for (size_t i = 0; i < elements_.size(); i++) {
- ASSERT_EQ(elements_[i], elements[i]);
- }
-}
-
-TEST(TestColumnDescriptor, TestAttrs) {
- NodePtr node = PrimitiveNode::Make("name", Repetition::OPTIONAL, Type::BYTE_ARRAY,
- LogicalType::UTF8);
- ColumnDescriptor descr(node, 4, 1);
-
- ASSERT_EQ("name", descr.name());
- ASSERT_EQ(4, descr.max_definition_level());
- ASSERT_EQ(1, descr.max_repetition_level());
-
- ASSERT_EQ(Type::BYTE_ARRAY, descr.physical_type());
-
- ASSERT_EQ(-1, descr.type_length());
-
- // Test FIXED_LEN_BYTE_ARRAY
- node = PrimitiveNode::Make("name", Repetition::OPTIONAL, Type::FIXED_LEN_BYTE_ARRAY,
- LogicalType::DECIMAL, 12, 10, 4);
- descr = ColumnDescriptor(node, 4, 1);
-
- ASSERT_EQ(Type::FIXED_LEN_BYTE_ARRAY, descr.physical_type());
- ASSERT_EQ(12, descr.type_length());
-}
-
-class TestSchemaDescriptor : public ::testing::Test {
- public:
- void setUp() {}
-
- protected:
- SchemaDescriptor descr_;
-};
-
-TEST_F(TestSchemaDescriptor, InitNonGroup) {
- NodePtr node = PrimitiveNode::Make("field", Repetition::OPTIONAL, Type::INT32);
-
- ASSERT_THROW(descr_.Init(node), ParquetException);
-}
-
-TEST_F(TestSchemaDescriptor, Equals) {
- NodePtr schema;
-
- NodePtr inta = Int32("a", Repetition::REQUIRED);
- NodePtr intb = Int64("b", Repetition::OPTIONAL);
- NodePtr intb2 = Int64("b2", Repetition::OPTIONAL);
- NodePtr intc = ByteArray("c", Repetition::REPEATED);
-
- NodePtr item1 = Int64("item1", Repetition::REQUIRED);
- NodePtr item2 = Boolean("item2", Repetition::OPTIONAL);
- NodePtr item3 = Int32("item3", Repetition::REPEATED);
- NodePtr list(GroupNode::Make("records", Repetition::REPEATED, {item1, item2, item3},
- LogicalType::LIST));
-
- NodePtr bag(GroupNode::Make("bag", Repetition::OPTIONAL, {list}));
- NodePtr bag2(GroupNode::Make("bag", Repetition::REQUIRED, {list}));
-
- SchemaDescriptor descr1;
- descr1.Init(GroupNode::Make("schema", Repetition::REPEATED, {inta, intb, intc, bag}));
-
- ASSERT_TRUE(descr1.Equals(descr1));
-
- SchemaDescriptor descr2;
- descr2.Init(GroupNode::Make("schema", Repetition::REPEATED, {inta, intb, intc, bag2}));
- ASSERT_FALSE(descr1.Equals(descr2));
-
- SchemaDescriptor descr3;
- descr3.Init(GroupNode::Make("schema", Repetition::REPEATED, {inta, intb2, intc, bag}));
- ASSERT_FALSE(descr1.Equals(descr3));
-
- // Robust to name of parent node
- SchemaDescriptor descr4;
- descr4.Init(GroupNode::Make("SCHEMA", Repetition::REPEATED, {inta, intb, intc, bag}));
- ASSERT_TRUE(descr1.Equals(descr4));
-
- SchemaDescriptor descr5;
- descr5.Init(
- GroupNode::Make("schema", Repetition::REPEATED, {inta, intb, intc, bag, intb2}));
- ASSERT_FALSE(descr1.Equals(descr5));
-
- // Different max repetition / definition levels
- ColumnDescriptor col1(inta, 5, 1);
- ColumnDescriptor col2(inta, 6, 1);
- ColumnDescriptor col3(inta, 5, 2);
-
- ASSERT_TRUE(col1.Equals(col1));
- ASSERT_FALSE(col1.Equals(col2));
- ASSERT_FALSE(col1.Equals(col3));
-}
-
-TEST_F(TestSchemaDescriptor, BuildTree) {
- NodeVector fields;
- NodePtr schema;
-
- NodePtr inta = Int32("a", Repetition::REQUIRED);
- fields.push_back(inta);
- fields.push_back(Int64("b", Repetition::OPTIONAL));
- fields.push_back(ByteArray("c", Repetition::REPEATED));
-
- // 3-level list encoding
- NodePtr item1 = Int64("item1", Repetition::REQUIRED);
- NodePtr item2 = Boolean("item2", Repetition::OPTIONAL);
- NodePtr item3 = Int32("item3", Repetition::REPEATED);
- NodePtr list(GroupNode::Make("records", Repetition::REPEATED, {item1, item2, item3},
- LogicalType::LIST));
- NodePtr bag(GroupNode::Make("bag", Repetition::OPTIONAL, {list}));
- fields.push_back(bag);
-
- schema = GroupNode::Make("schema", Repetition::REPEATED, fields);
-
- descr_.Init(schema);
-
- int nleaves = 6;
-
- // 6 leaves
- ASSERT_EQ(nleaves, descr_.num_columns());
-
- // mdef mrep
- // required int32 a 0 0
- // optional int64 b 1 0
- // repeated byte_array c 1 1
- // optional group bag 1 0
- // repeated group records 2 1
- // required int64 item1 2 1
- // optional boolean item2 3 1
- // repeated int32 item3 3 2
- int16_t ex_max_def_levels[6] = {0, 1, 1, 2, 3, 3};
- int16_t ex_max_rep_levels[6] = {0, 0, 1, 1, 1, 2};
-
- for (int i = 0; i < nleaves; ++i) {
- const ColumnDescriptor* col = descr_.Column(i);
- EXPECT_EQ(ex_max_def_levels[i], col->max_definition_level()) << i;
- EXPECT_EQ(ex_max_rep_levels[i], col->max_repetition_level()) << i;
- }
-
- ASSERT_EQ(descr_.Column(0)->path()->ToDotString(), "a");
- ASSERT_EQ(descr_.Column(1)->path()->ToDotString(), "b");
- ASSERT_EQ(descr_.Column(2)->path()->ToDotString(), "c");
- ASSERT_EQ(descr_.Column(3)->path()->ToDotString(), "bag.records.item1");
- ASSERT_EQ(descr_.Column(4)->path()->ToDotString(), "bag.records.item2");
- ASSERT_EQ(descr_.Column(5)->path()->ToDotString(), "bag.records.item3");
-
- for (int i = 0; i < nleaves; ++i) {
- auto col = descr_.Column(i);
- ASSERT_EQ(i, descr_.ColumnIndex(*col->schema_node()));
- }
-
- // Test non-column nodes find
- NodePtr non_column_alien = Int32("alien", Repetition::REQUIRED); // other path
- NodePtr non_column_familiar = Int32("a", Repetition::REPEATED); // other node
- ASSERT_TRUE(descr_.ColumnIndex(*non_column_alien) < 0);
- ASSERT_TRUE(descr_.ColumnIndex(*non_column_familiar) < 0);
-
- ASSERT_EQ(inta.get(), descr_.GetColumnRoot(0));
- ASSERT_EQ(bag.get(), descr_.GetColumnRoot(3));
- ASSERT_EQ(bag.get(), descr_.GetColumnRoot(4));
- ASSERT_EQ(bag.get(), descr_.GetColumnRoot(5));
-
- ASSERT_EQ(schema.get(), descr_.group_node());
-
- // Init clears the leaves
- descr_.Init(schema);
- ASSERT_EQ(nleaves, descr_.num_columns());
-}
-
-static std::string Print(const NodePtr& node) {
- std::stringstream ss;
- PrintSchema(node.get(), ss);
- return ss.str();
-}
-
-TEST(TestSchemaPrinter, Examples) {
- // Test schema 1
- NodeVector fields;
- fields.push_back(Int32("a", Repetition::REQUIRED));
-
- // 3-level list encoding
- NodePtr item1 = Int64("item1");
- NodePtr item2 = Boolean("item2", Repetition::REQUIRED);
- NodePtr list(
- GroupNode::Make("b", Repetition::REPEATED, {item1, item2}, LogicalType::LIST));
- NodePtr bag(GroupNode::Make("bag", Repetition::OPTIONAL, {list}));
- fields.push_back(bag);
-
- fields.push_back(PrimitiveNode::Make("c", Repetition::REQUIRED, Type::INT32,
- LogicalType::DECIMAL, -1, 3, 2));
-
- NodePtr schema = GroupNode::Make("schema", Repetition::REPEATED, fields);
-
- std::string result = Print(schema);
- std::string expected = R"(message schema {
- required int32 a;
- optional group bag {
- repeated group b (LIST) {
- optional int64 item1;
- required boolean item2;
- }
- }
- required int32 c (DECIMAL(3,2));
-}
-)";
- ASSERT_EQ(expected, result);
-}
-
-} // namespace schema
-} // namespace parquet
diff --git a/src/parquet/schema.cc b/src/parquet/schema.cc
deleted file mode 100644
index 7718781..0000000
--- a/src/parquet/schema.cc
+++ /dev/null
@@ -1,756 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#include "parquet/schema.h"
-#include "parquet/schema-internal.h"
-
-#include <algorithm>
-#include <memory>
-#include <sstream>
-#include <string>
-
-#include "parquet/exception.h"
-#include "parquet/parquet_types.h"
-#include "parquet/thrift.h"
-
-using parquet::format::SchemaElement;
-
-namespace parquet {
-
-namespace schema {
-
-// ----------------------------------------------------------------------
-// ColumnPath
-
-std::shared_ptr<ColumnPath> ColumnPath::FromDotString(const std::string& dotstring) {
- std::stringstream ss(dotstring);
- std::string item;
- std::vector<std::string> path;
- while (std::getline(ss, item, '.')) {
- path.push_back(item);
- }
- return std::shared_ptr<ColumnPath>(new ColumnPath(std::move(path)));
-}
-
-std::shared_ptr<ColumnPath> ColumnPath::FromNode(const Node& node) {
- // Build the path in reverse order as we traverse the nodes to the top
- std::vector<std::string> rpath_;
- const Node* cursor = &node;
- // The schema node is not part of the ColumnPath
- while (cursor->parent()) {
- rpath_.push_back(cursor->name());
- cursor = cursor->parent();
- }
-
- // Build ColumnPath in correct order
- std::vector<std::string> path(rpath_.crbegin(), rpath_.crend());
- return std::make_shared<ColumnPath>(std::move(path));
-}
-
-std::shared_ptr<ColumnPath> ColumnPath::extend(const std::string& node_name) const {
- std::vector<std::string> path;
- path.reserve(path_.size() + 1);
- path.resize(path_.size() + 1);
- std::copy(path_.cbegin(), path_.cend(), path.begin());
- path[path_.size()] = node_name;
-
- return std::shared_ptr<ColumnPath>(new ColumnPath(std::move(path)));
-}
-
-std::string ColumnPath::ToDotString() const {
- std::stringstream ss;
- for (auto it = path_.cbegin(); it != path_.cend(); ++it) {
- if (it != path_.cbegin()) {
- ss << ".";
- }
- ss << *it;
- }
- return ss.str();
-}
-
-const std::vector<std::string>& ColumnPath::ToDotVector() const { return path_; }
-
-// ----------------------------------------------------------------------
-// Base node
-
-const std::shared_ptr<ColumnPath> Node::path() const {
- // TODO(itaiin): Cache the result, or more precisely, cache ->ToDotString()
- // since it is being used to access the leaf nodes
- return ColumnPath::FromNode(*this);
-}
-
-bool Node::EqualsInternal(const Node* other) const {
- return type_ == other->type_ && name_ == other->name_ &&
- repetition_ == other->repetition_ && logical_type_ == other->logical_type_;
-}
-
-void Node::SetParent(const Node* parent) { parent_ = parent; }
-
-// ----------------------------------------------------------------------
-// Primitive node
-
-PrimitiveNode::PrimitiveNode(const std::string& name, Repetition::type repetition,
- Type::type type, LogicalType::type logical_type, int length,
- int precision, int scale, int id)
- : Node(Node::PRIMITIVE, name, repetition, logical_type, id),
- physical_type_(type),
- type_length_(length) {
- std::stringstream ss;
-
- // PARQUET-842: In an earlier revision, decimal_metadata_.isset was being
- // set to true, but Impala will raise an incompatible metadata in such cases
- memset(&decimal_metadata_, 0, sizeof(decimal_metadata_));
-
- // Check if the physical and logical types match
- // Mapping referred from Apache parquet-mr as on 2016-02-22
- switch (logical_type) {
- case LogicalType::NONE:
- // Logical type not set
- break;
- case LogicalType::UTF8:
- case LogicalType::JSON:
- case LogicalType::BSON:
- if (type != Type::BYTE_ARRAY) {
- ss << LogicalTypeToString(logical_type);
- ss << " can only annotate BYTE_ARRAY fields";
- throw ParquetException(ss.str());
- }
- break;
- case LogicalType::DECIMAL:
- if ((type != Type::INT32) && (type != Type::INT64) && (type != Type::BYTE_ARRAY) &&
- (type != Type::FIXED_LEN_BYTE_ARRAY)) {
- ss << "DECIMAL can only annotate INT32, INT64, BYTE_ARRAY, and FIXED";
- throw ParquetException(ss.str());
- }
- if (precision <= 0) {
- ss << "Invalid DECIMAL precision: " << precision
- << ". Precision must be a number between 1 and 38 inclusive";
- throw ParquetException(ss.str());
- }
- if (scale < 0) {
- ss << "Invalid DECIMAL scale: " << scale
- << ". Scale must be a number between 0 and precision inclusive";
- throw ParquetException(ss.str());
- }
- if (scale > precision) {
- ss << "Invalid DECIMAL scale " << scale;
- ss << " cannot be greater than precision " << precision;
- throw ParquetException(ss.str());
- }
- decimal_metadata_.isset = true;
- decimal_metadata_.precision = precision;
- decimal_metadata_.scale = scale;
- break;
- case LogicalType::DATE:
- case LogicalType::TIME_MILLIS:
- case LogicalType::UINT_8:
- case LogicalType::UINT_16:
- case LogicalType::UINT_32:
- case LogicalType::INT_8:
- case LogicalType::INT_16:
- case LogicalType::INT_32:
- if (type != Type::INT32) {
- ss << LogicalTypeToString(logical_type);
- ss << " can only annotate INT32";
- throw ParquetException(ss.str());
- }
- break;
- case LogicalType::TIME_MICROS:
- case LogicalType::TIMESTAMP_MILLIS:
- case LogicalType::TIMESTAMP_MICROS:
- case LogicalType::UINT_64:
- case LogicalType::INT_64:
- if (type != Type::INT64) {
- ss << LogicalTypeToString(logical_type);
- ss << " can only annotate INT64";
- throw ParquetException(ss.str());
- }
- break;
- case LogicalType::INTERVAL:
- if ((type != Type::FIXED_LEN_BYTE_ARRAY) || (length != 12)) {
- ss << "INTERVAL can only annotate FIXED_LEN_BYTE_ARRAY(12)";
- throw ParquetException(ss.str());
- }
- break;
- case LogicalType::ENUM:
- if (type != Type::BYTE_ARRAY) {
- ss << "ENUM can only annotate BYTE_ARRAY fields";
- throw ParquetException(ss.str());
- }
- break;
- case LogicalType::NA:
- // NA can annotate any type
- break;
- default:
- ss << LogicalTypeToString(logical_type);
- ss << " can not be applied to a primitive type";
- throw ParquetException(ss.str());
- }
- if (type == Type::FIXED_LEN_BYTE_ARRAY) {
- if (length <= 0) {
- ss << "Invalid FIXED_LEN_BYTE_ARRAY length: " << length;
- throw ParquetException(ss.str());
- }
- type_length_ = length;
- }
-}
-
-bool PrimitiveNode::EqualsInternal(const PrimitiveNode* other) const {
- bool is_equal = true;
- if ((physical_type_ != other->physical_type_) ||
- (logical_type_ != other->logical_type_)) {
- return false;
- }
- if (logical_type_ == LogicalType::DECIMAL) {
- is_equal &= (decimal_metadata_.precision == other->decimal_metadata_.precision) &&
- (decimal_metadata_.scale == other->decimal_metadata_.scale);
- }
- if (physical_type_ == Type::FIXED_LEN_BYTE_ARRAY) {
- is_equal &= (type_length_ == other->type_length_);
- }
- return is_equal;
-}
-
-bool PrimitiveNode::Equals(const Node* other) const {
- if (!Node::EqualsInternal(other)) {
- return false;
- }
- return EqualsInternal(static_cast<const PrimitiveNode*>(other));
-}
-
-void PrimitiveNode::Visit(Node::Visitor* visitor) { visitor->Visit(this); }
-
-void PrimitiveNode::VisitConst(Node::ConstVisitor* visitor) const {
- visitor->Visit(this);
-}
-
-// ----------------------------------------------------------------------
-// Group node
-
-bool GroupNode::EqualsInternal(const GroupNode* other) const {
- if (this == other) {
- return true;
- }
- if (this->field_count() != other->field_count()) {
- return false;
- }
- for (int i = 0; i < this->field_count(); ++i) {
- if (!this->field(i)->Equals(other->field(i).get())) {
- return false;
- }
- }
- return true;
-}
-
-bool GroupNode::Equals(const Node* other) const {
- if (!Node::EqualsInternal(other)) {
- return false;
- }
- return EqualsInternal(static_cast<const GroupNode*>(other));
-}
-
-int GroupNode::FieldIndex(const std::string& name) const {
- auto search = field_name_to_idx_.find(name);
- if (search == field_name_to_idx_.end()) {
- // Not found
- return -1;
- }
- return search->second;
-}
-
-int GroupNode::FieldIndex(const Node& node) const {
- auto search = field_name_to_idx_.equal_range(node.name());
- for (auto it = search.first; it != search.second; ++it) {
- const int idx = it->second;
- if (&node == field(idx).get()) {
- return idx;
- }
- }
- return -1;
-}
-
-void GroupNode::Visit(Node::Visitor* visitor) { visitor->Visit(this); }
-
-void GroupNode::VisitConst(Node::ConstVisitor* visitor) const { visitor->Visit(this); }
-
-// ----------------------------------------------------------------------
-// Node construction from Parquet metadata
-
-struct NodeParams {
- explicit NodeParams(const std::string& name) : name(name) {}
-
- const std::string& name;
- Repetition::type repetition;
- LogicalType::type logical_type;
-};
-
-static inline NodeParams GetNodeParams(const format::SchemaElement* element) {
- NodeParams params(element->name);
-
- params.repetition = FromThrift(element->repetition_type);
- if (element->__isset.converted_type) {
- params.logical_type = FromThrift(element->converted_type);
- } else {
- params.logical_type = LogicalType::NONE;
- }
- return params;
-}
-
-std::unique_ptr<Node> GroupNode::FromParquet(const void* opaque_element, int node_id,
- const NodeVector& fields) {
- const format::SchemaElement* element =
- static_cast<const format::SchemaElement*>(opaque_element);
- NodeParams params = GetNodeParams(element);
- return std::unique_ptr<Node>(new GroupNode(params.name, params.repetition, fields,
- params.logical_type, node_id));
-}
-
-std::unique_ptr<Node> PrimitiveNode::FromParquet(const void* opaque_element,
- int node_id) {
- const format::SchemaElement* element =
- static_cast<const format::SchemaElement*>(opaque_element);
- NodeParams params = GetNodeParams(element);
-
- std::unique_ptr<PrimitiveNode> result =
- std::unique_ptr<PrimitiveNode>(new PrimitiveNode(
- params.name, params.repetition, FromThrift(element->type), params.logical_type,
- element->type_length, element->precision, element->scale, node_id));
-
- // Return as unique_ptr to the base type
- return std::unique_ptr<Node>(result.release());
-}
-
-void GroupNode::ToParquet(void* opaque_element) const {
- format::SchemaElement* element = static_cast<format::SchemaElement*>(opaque_element);
- element->__set_name(name_);
- element->__set_num_children(field_count());
- element->__set_repetition_type(ToThrift(repetition_));
- if (logical_type_ != LogicalType::NONE) {
- element->__set_converted_type(ToThrift(logical_type_));
- }
-}
-
-void PrimitiveNode::ToParquet(void* opaque_element) const {
- format::SchemaElement* element = static_cast<format::SchemaElement*>(opaque_element);
-
- element->__set_name(name_);
- element->__set_repetition_type(ToThrift(repetition_));
- if (logical_type_ != LogicalType::NONE) {
- element->__set_converted_type(ToThrift(logical_type_));
- }
- element->__set_type(ToThrift(physical_type_));
- if (physical_type_ == Type::FIXED_LEN_BYTE_ARRAY) {
- element->__set_type_length(type_length_);
- }
- if (decimal_metadata_.isset) {
- element->__set_precision(decimal_metadata_.precision);
- element->__set_scale(decimal_metadata_.scale);
- }
-}
-
-// ----------------------------------------------------------------------
-// Schema converters
-
-std::unique_ptr<Node> FlatSchemaConverter::Convert() {
- const SchemaElement& root = elements_[0];
-
- // Validate the root node
- if (root.num_children == 0) {
- throw ParquetException("Root node did not have children");
- }
-
- // Relaxing this restriction as some implementations don't set this
- // if (root.repetition_type != FieldRepetitionType::REPEATED) {
- // throw ParquetException("Root node was not FieldRepetitionType::REPEATED");
- // }
-
- return NextNode();
-}
-
-std::unique_ptr<Node> FlatSchemaConverter::NextNode() {
- const SchemaElement& element = Next();
-
- int node_id = next_id();
-
- const void* opaque_element = static_cast<const void*>(&element);
-
- if (element.num_children == 0) {
- // Leaf (primitive) node
- return PrimitiveNode::FromParquet(opaque_element, node_id);
- } else {
- // Group
- NodeVector fields;
- for (int i = 0; i < element.num_children; ++i) {
- std::unique_ptr<Node> field = NextNode();
- fields.push_back(NodePtr(field.release()));
- }
- return GroupNode::FromParquet(opaque_element, node_id, fields);
- }
-}
-
-const format::SchemaElement& FlatSchemaConverter::Next() {
- if (pos_ == length_) {
- throw ParquetException("Malformed schema: not enough SchemaElement values");
- }
- return elements_[pos_++];
-}
-
-std::shared_ptr<SchemaDescriptor> FromParquet(const std::vector<SchemaElement>& schema) {
- FlatSchemaConverter converter(&schema[0], static_cast<int>(schema.size()));
- std::unique_ptr<Node> root = converter.Convert();
-
- std::shared_ptr<SchemaDescriptor> descr = std::make_shared<SchemaDescriptor>();
- descr->Init(std::shared_ptr<GroupNode>(static_cast<GroupNode*>(root.release())));
-
- return descr;
-}
-
-void ToParquet(const GroupNode* schema, std::vector<format::SchemaElement>* out) {
- SchemaFlattener flattener(schema, out);
- flattener.Flatten();
-}
-
-class SchemaVisitor : public Node::ConstVisitor {
- public:
- explicit SchemaVisitor(std::vector<format::SchemaElement>* elements)
- : elements_(elements) {}
-
- void Visit(const Node* node) override {
- format::SchemaElement element;
- node->ToParquet(&element);
- elements_->push_back(element);
-
- if (node->is_group()) {
- const GroupNode* group_node = static_cast<const GroupNode*>(node);
- for (int i = 0; i < group_node->field_count(); ++i) {
- group_node->field(i)->VisitConst(this);
- }
- }
- }
-
- private:
- std::vector<format::SchemaElement>* elements_;
-};
-
-SchemaFlattener::SchemaFlattener(const GroupNode* schema,
- std::vector<format::SchemaElement>* out)
- : root_(schema), elements_(out) {}
-
-void SchemaFlattener::Flatten() {
- SchemaVisitor visitor(elements_);
- root_->VisitConst(&visitor);
-}
-
-// ----------------------------------------------------------------------
-// Schema printing
-
-class SchemaPrinter : public Node::ConstVisitor {
- public:
- explicit SchemaPrinter(std::ostream& stream, int indent_width)
- : stream_(stream), indent_(0), indent_width_(2) {}
-
- void Visit(const Node* node) override;
-
- private:
- void Visit(const PrimitiveNode* node);
- void Visit(const GroupNode* node);
-
- void Indent();
-
- std::ostream& stream_;
-
- int indent_;
- int indent_width_;
-};
-
-static void PrintRepLevel(Repetition::type repetition, std::ostream& stream) {
- switch (repetition) {
- case Repetition::REQUIRED:
- stream << "required";
- break;
- case Repetition::OPTIONAL:
- stream << "optional";
- break;
- case Repetition::REPEATED:
- stream << "repeated";
- break;
- default:
- break;
- }
-}
-
-static void PrintType(const PrimitiveNode* node, std::ostream& stream) {
- switch (node->physical_type()) {
- case Type::BOOLEAN:
- stream << "boolean";
- break;
- case Type::INT32:
- stream << "int32";
- break;
- case Type::INT64:
- stream << "int64";
- break;
- case Type::INT96:
- stream << "int96";
- break;
- case Type::FLOAT:
- stream << "float";
- break;
- case Type::DOUBLE:
- stream << "double";
- break;
- case Type::BYTE_ARRAY:
- stream << "binary";
- break;
- case Type::FIXED_LEN_BYTE_ARRAY:
- stream << "fixed_len_byte_array(" << node->type_length() << ")";
- break;
- default:
- break;
- }
-}
-
-static void PrintLogicalType(const PrimitiveNode* node, std::ostream& stream) {
- auto lt = node->logical_type();
- if (lt == LogicalType::DECIMAL) {
- stream << " (" << LogicalTypeToString(lt) << "(" << node->decimal_metadata().precision
- << "," << node->decimal_metadata().scale << "))";
- } else if (lt != LogicalType::NONE) {
- stream << " (" << LogicalTypeToString(lt) << ")";
- }
-}
-
-void SchemaPrinter::Visit(const PrimitiveNode* node) {
- PrintRepLevel(node->repetition(), stream_);
- stream_ << " ";
- PrintType(node, stream_);
- stream_ << " " << node->name();
- PrintLogicalType(node, stream_);
- stream_ << ";" << std::endl;
-}
-
-void SchemaPrinter::Visit(const GroupNode* node) {
- if (!node->parent()) {
- stream_ << "message " << node->name() << " {" << std::endl;
- } else {
- PrintRepLevel(node->repetition(), stream_);
- stream_ << " group " << node->name();
- auto lt = node->logical_type();
- if (lt != LogicalType::NONE) {
- stream_ << " (" << LogicalTypeToString(lt) << ")";
- }
- stream_ << " {" << std::endl;
- }
-
- indent_ += indent_width_;
- for (int i = 0; i < node->field_count(); ++i) {
- node->field(i)->VisitConst(this);
- }
- indent_ -= indent_width_;
- Indent();
- stream_ << "}" << std::endl;
-}
-
-void SchemaPrinter::Indent() {
- if (indent_ > 0) {
- std::string spaces(indent_, ' ');
- stream_ << spaces;
- }
-}
-
-void SchemaPrinter::Visit(const Node* node) {
- Indent();
- if (node->is_group()) {
- Visit(static_cast<const GroupNode*>(node));
- } else {
- // Primitive
- Visit(static_cast<const PrimitiveNode*>(node));
- }
-}
-
-void PrintSchema(const Node* schema, std::ostream& stream, int indent_width) {
- SchemaPrinter printer(stream, indent_width);
- printer.Visit(schema);
-}
-
-} // namespace schema
-
-using schema::ColumnPath;
-using schema::GroupNode;
-using schema::Node;
-using schema::NodePtr;
-using schema::PrimitiveNode;
-
-void SchemaDescriptor::Init(std::unique_ptr<schema::Node> schema) {
- Init(NodePtr(schema.release()));
-}
-
-class SchemaUpdater : public Node::Visitor {
- public:
- explicit SchemaUpdater(const std::vector<ColumnOrder>& column_orders)
- : column_orders_(column_orders), leaf_count_(0) {}
-
- void Visit(Node* node) override {
- if (node->is_group()) {
- GroupNode* group_node = static_cast<GroupNode*>(node);
- for (int i = 0; i < group_node->field_count(); ++i) {
- group_node->field(i)->Visit(this);
- }
- } else { // leaf node
- PrimitiveNode* leaf_node = static_cast<PrimitiveNode*>(node);
- leaf_node->SetColumnOrder(column_orders_[leaf_count_++]);
- }
- }
-
- private:
- const std::vector<ColumnOrder>& column_orders_;
- int leaf_count_;
-};
-
-void SchemaDescriptor::updateColumnOrders(const std::vector<ColumnOrder>& column_orders) {
- if (static_cast<int>(column_orders.size()) != num_columns()) {
- throw ParquetException("Malformed schema: not enough ColumnOrder values");
- }
- SchemaUpdater visitor(column_orders);
- const_cast<GroupNode*>(group_node_)->Visit(&visitor);
-}
-
-void SchemaDescriptor::Init(const NodePtr& schema) {
- schema_ = schema;
-
- if (!schema_->is_group()) {
- throw ParquetException("Must initialize with a schema group");
- }
-
- group_node_ = static_cast<const GroupNode*>(schema_.get());
- leaves_.clear();
-
- for (int i = 0; i < group_node_->field_count(); ++i) {
- BuildTree(group_node_->field(i), 0, 0, group_node_->field(i));
- }
-}
-
-bool SchemaDescriptor::Equals(const SchemaDescriptor& other) const {
- if (this->num_columns() != other.num_columns()) {
- return false;
- }
-
- for (int i = 0; i < this->num_columns(); ++i) {
- if (!this->Column(i)->Equals(*other.Column(i))) {
- return false;
- }
- }
-
- return true;
-}
-
-void SchemaDescriptor::BuildTree(const NodePtr& node, int16_t max_def_level,
- int16_t max_rep_level, const NodePtr& base) {
- if (node->is_optional()) {
- ++max_def_level;
- } else if (node->is_repeated()) {
- // Repeated fields add a definition level. This is used to distinguish
- // between an empty list and a list with an item in it.
- ++max_rep_level;
- ++max_def_level;
- }
-
- // Now, walk the schema and create a ColumnDescriptor for each leaf node
- if (node->is_group()) {
- const GroupNode* group = static_cast<const GroupNode*>(node.get());
- for (int i = 0; i < group->field_count(); ++i) {
- BuildTree(group->field(i), max_def_level, max_rep_level, base);
- }
- } else {
- // Primitive node, append to leaves
- leaves_.push_back(ColumnDescriptor(node, max_def_level, max_rep_level, this));
- leaf_to_base_.emplace(static_cast<int>(leaves_.size()) - 1, base);
- leaf_to_idx_.emplace(node->path()->ToDotString(),
- static_cast<int>(leaves_.size()) - 1);
- }
-}
-
-ColumnDescriptor::ColumnDescriptor(const schema::NodePtr& node,
- int16_t max_definition_level,
- int16_t max_repetition_level,
- const SchemaDescriptor* schema_descr)
- : node_(node),
- max_definition_level_(max_definition_level),
- max_repetition_level_(max_repetition_level) {
- if (!node_->is_primitive()) {
- throw ParquetException("Must be a primitive type");
- }
- primitive_node_ = static_cast<const PrimitiveNode*>(node_.get());
-}
-
-bool ColumnDescriptor::Equals(const ColumnDescriptor& other) const {
- return primitive_node_->Equals(other.primitive_node_) &&
- max_repetition_level() == other.max_repetition_level() &&
- max_definition_level() == other.max_definition_level();
-}
-
-const ColumnDescriptor* SchemaDescriptor::Column(int i) const {
- DCHECK(i >= 0 && i < static_cast<int>(leaves_.size()));
- return &leaves_[i];
-}
-
-int SchemaDescriptor::ColumnIndex(const std::string& node_path) const {
- auto search = leaf_to_idx_.find(node_path);
- if (search == leaf_to_idx_.end()) {
- // Not found
- return -1;
- }
- return search->second;
-}
-
-int SchemaDescriptor::ColumnIndex(const Node& node) const {
- auto search = leaf_to_idx_.equal_range(node.path()->ToDotString());
- for (auto it = search.first; it != search.second; ++it) {
- const int idx = it->second;
- if (&node == Column(idx)->schema_node().get()) {
- return idx;
- }
- }
- return -1;
-}
-
-const schema::Node* SchemaDescriptor::GetColumnRoot(int i) const {
- DCHECK(i >= 0 && i < static_cast<int>(leaves_.size()));
- return leaf_to_base_.find(i)->second.get();
-}
-
-std::string SchemaDescriptor::ToString() const {
- std::ostringstream ss;
- PrintSchema(schema_.get(), ss);
- return ss.str();
-}
-
-int ColumnDescriptor::type_scale() const {
- return primitive_node_->decimal_metadata().scale;
-}
-
-int ColumnDescriptor::type_precision() const {
- return primitive_node_->decimal_metadata().precision;
-}
-
-int ColumnDescriptor::type_length() const { return primitive_node_->type_length(); }
-
-const std::shared_ptr<ColumnPath> ColumnDescriptor::path() const {
- return primitive_node_->path();
-}
-
-} // namespace parquet
diff --git a/src/parquet/schema.h b/src/parquet/schema.h
deleted file mode 100644
index 1a94ed1..0000000
--- a/src/parquet/schema.h
+++ /dev/null
@@ -1,452 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-// This module contains the logical parquet-cpp types (independent of Thrift
-// structures), schema nodes, and related type tools
-
-#ifndef PARQUET_SCHEMA_TYPES_H
-#define PARQUET_SCHEMA_TYPES_H
-
-#include <cstdint>
-#include <memory>
-#include <ostream>
-#include <string>
-#include <unordered_map>
-#include <vector>
-
-#include "parquet/types.h"
-#include "parquet/util/macros.h"
-#include "parquet/util/visibility.h"
-
-namespace parquet {
-
-class SchemaDescriptor;
-
-namespace schema {
-
-class Node;
-
-// List encodings: using the terminology from Impala to define different styles
-// of representing logical lists (a.k.a. ARRAY types) in Parquet schemas. Since
-// the converted type named in the Parquet metadata is ConvertedType::LIST we
-// use that terminology here. It also helps distinguish from the *_ARRAY
-// primitive types.
-//
-// One-level encoding: Only allows required lists with required cells
-// repeated value_type name
-//
-// Two-level encoding: Enables optional lists with only required cells
-// <required/optional> group list
-// repeated value_type item
-//
-// Three-level encoding: Enables optional lists with optional cells
-// <required/optional> group bag
-// repeated group list
-// <required/optional> value_type item
-//
-// 2- and 1-level encoding are respectively equivalent to 3-level encoding with
-// the non-repeated nodes set to required.
-//
-// The "official" encoding recommended in the Parquet spec is the 3-level, and
-// we use that as the default when creating list types. For semantic completeness
-// we allow the other two. Since all types of encodings will occur "in the
-// wild" we need to be able to interpret the associated definition levels in
-// the context of the actual encoding used in the file.
-//
-// NB: Some Parquet writers may not set ConvertedType::LIST on the repeated
-// SchemaElement, which could make things challenging if we are trying to infer
-// that a sequence of nodes semantically represents an array according to one
-// of these encodings (versus a struct containing an array). We should refuse
-// the temptation to guess, as they say.
-struct ListEncoding {
- enum type { ONE_LEVEL, TWO_LEVEL, THREE_LEVEL };
-};
-
-struct DecimalMetadata {
- bool isset;
- int32_t scale;
- int32_t precision;
-};
-
-class PARQUET_EXPORT ColumnPath {
- public:
- ColumnPath() : path_() {}
- explicit ColumnPath(const std::vector<std::string>& path) : path_(path) {}
- explicit ColumnPath(std::vector<std::string>&& path) : path_(path) {}
-
- static std::shared_ptr<ColumnPath> FromDotString(const std::string& dotstring);
- static std::shared_ptr<ColumnPath> FromNode(const Node& node);
-
- std::shared_ptr<ColumnPath> extend(const std::string& node_name) const;
- std::string ToDotString() const;
- const std::vector<std::string>& ToDotVector() const;
-
- protected:
- std::vector<std::string> path_;
-};
-
-class GroupNode;
-
-// Base class for logical schema types. A type has a name, repetition level,
-// and optionally a logical type (ConvertedType in Parquet metadata parlance)
-class PARQUET_EXPORT Node {
- public:
- enum type { PRIMITIVE, GROUP };
-
- Node(Node::type type, const std::string& name, Repetition::type repetition,
- LogicalType::type logical_type = LogicalType::NONE, int id = -1)
- : type_(type),
- name_(name),
- repetition_(repetition),
- logical_type_(logical_type),
- id_(id),
- parent_(nullptr) {}
-
- virtual ~Node() {}
-
- bool is_primitive() const { return type_ == Node::PRIMITIVE; }
-
- bool is_group() const { return type_ == Node::GROUP; }
-
- bool is_optional() const { return repetition_ == Repetition::OPTIONAL; }
-
- bool is_repeated() const { return repetition_ == Repetition::REPEATED; }
-
- bool is_required() const { return repetition_ == Repetition::REQUIRED; }
-
- virtual bool Equals(const Node* other) const = 0;
-
- const std::string& name() const { return name_; }
-
- Node::type node_type() const { return type_; }
-
- Repetition::type repetition() const { return repetition_; }
-
- LogicalType::type logical_type() const { return logical_type_; }
-
- int id() const { return id_; }
-
- const Node* parent() const { return parent_; }
-
- const std::shared_ptr<ColumnPath> path() const;
-
- // ToParquet returns an opaque void* to avoid exporting
- // parquet::SchemaElement into the public API
- virtual void ToParquet(void* opaque_element) const = 0;
-
- // Node::Visitor abstract class for walking schemas with the visitor pattern
- class Visitor {
- public:
- virtual ~Visitor() {}
-
- virtual void Visit(Node* node) = 0;
- };
- class ConstVisitor {
- public:
- virtual ~ConstVisitor() {}
-
- virtual void Visit(const Node* node) = 0;
- };
-
- virtual void Visit(Visitor* visitor) = 0;
- virtual void VisitConst(ConstVisitor* visitor) const = 0;
-
- protected:
- friend class GroupNode;
-
- Node::type type_;
- std::string name_;
- Repetition::type repetition_;
- LogicalType::type logical_type_;
- int id_;
- // Nodes should not be shared, they have a single parent.
- const Node* parent_;
-
- bool EqualsInternal(const Node* other) const;
- void SetParent(const Node* p_parent);
-
- private:
- DISALLOW_COPY_AND_ASSIGN(Node);
-};
-
-// Save our breath all over the place with these typedefs
-typedef std::shared_ptr<Node> NodePtr;
-typedef std::vector<NodePtr> NodeVector;
-
-// A type that is one of the primitive Parquet storage types. In addition to
-// the other type metadata (name, repetition level, logical type), also has the
-// physical storage type and their type-specific metadata (byte width, decimal
-// parameters)
-class PARQUET_EXPORT PrimitiveNode : public Node {
- public:
- // FromParquet accepts an opaque void* to avoid exporting
- // parquet::SchemaElement into the public API
- static std::unique_ptr<Node> FromParquet(const void* opaque_element, int id);
-
- static inline NodePtr Make(const std::string& name, Repetition::type repetition,
- Type::type type,
- LogicalType::type logical_type = LogicalType::NONE,
- int length = -1, int precision = -1, int scale = -1) {
- return NodePtr(new PrimitiveNode(name, repetition, type, logical_type, length,
- precision, scale));
- }
-
- bool Equals(const Node* other) const override;
-
- Type::type physical_type() const { return physical_type_; }
-
- ColumnOrder column_order() const { return column_order_; }
-
- void SetColumnOrder(ColumnOrder column_order) { column_order_ = column_order; }
-
- int32_t type_length() const { return type_length_; }
-
- const DecimalMetadata& decimal_metadata() const { return decimal_metadata_; }
-
- void ToParquet(void* opaque_element) const override;
- void Visit(Visitor* visitor) override;
- void VisitConst(ConstVisitor* visitor) const override;
-
- private:
- PrimitiveNode(const std::string& name, Repetition::type repetition, Type::type type,
- LogicalType::type logical_type = LogicalType::NONE, int length = -1,
- int precision = -1, int scale = -1, int id = -1);
-
- Type::type physical_type_;
- int32_t type_length_;
- DecimalMetadata decimal_metadata_;
- ColumnOrder column_order_;
-
- // For FIXED_LEN_BYTE_ARRAY
- void SetTypeLength(int32_t length) { type_length_ = length; }
-
- // For Decimal logical type: Precision and scale
- void SetDecimalMetadata(int32_t scale, int32_t precision) {
- decimal_metadata_.scale = scale;
- decimal_metadata_.precision = precision;
- }
-
- bool EqualsInternal(const PrimitiveNode* other) const;
-
- FRIEND_TEST(TestPrimitiveNode, Attrs);
- FRIEND_TEST(TestPrimitiveNode, Equals);
- FRIEND_TEST(TestPrimitiveNode, PhysicalLogicalMapping);
- FRIEND_TEST(TestPrimitiveNode, FromParquet);
-};
-
-class PARQUET_EXPORT GroupNode : public Node {
- public:
- // Like PrimitiveNode, GroupNode::FromParquet accepts an opaque void* to avoid exporting
- // parquet::SchemaElement into the public API
- static std::unique_ptr<Node> FromParquet(const void* opaque_element, int id,
- const NodeVector& fields);
-
- static inline NodePtr Make(const std::string& name, Repetition::type repetition,
- const NodeVector& fields,
- LogicalType::type logical_type = LogicalType::NONE) {
- return NodePtr(new GroupNode(name, repetition, fields, logical_type));
- }
-
- bool Equals(const Node* other) const override;
-
- NodePtr field(int i) const { return fields_[i]; }
- // Get the index of a field by its name, or negative value if not found.
- // If several fields share the same name, it is unspecified which one
- // is returned.
- int FieldIndex(const std::string& name) const;
- // Get the index of a field by its node, or negative value if not found.
- int FieldIndex(const Node& node) const;
-
- int field_count() const { return static_cast<int>(fields_.size()); }
-
- void ToParquet(void* opaque_element) const override;
- void Visit(Visitor* visitor) override;
- void VisitConst(ConstVisitor* visitor) const override;
-
- private:
- GroupNode(const std::string& name, Repetition::type repetition,
- const NodeVector& fields, LogicalType::type logical_type = LogicalType::NONE,
- int id = -1)
- : Node(Node::GROUP, name, repetition, logical_type, id), fields_(fields) {
- field_name_to_idx_.clear();
- auto field_idx = 0;
- for (NodePtr& field : fields_) {
- field->SetParent(this);
- field_name_to_idx_.emplace(field->name(), field_idx++);
- }
- }
-
- NodeVector fields_;
- bool EqualsInternal(const GroupNode* other) const;
-
- // Mapping between field name to the field index
- std::unordered_multimap<std::string, int> field_name_to_idx_;
-
- FRIEND_TEST(TestGroupNode, Attrs);
- FRIEND_TEST(TestGroupNode, Equals);
- FRIEND_TEST(TestGroupNode, FieldIndex);
- FRIEND_TEST(TestGroupNode, FieldIndexDuplicateName);
-};
-
-// ----------------------------------------------------------------------
-// Convenience primitive type factory functions
-
-#define PRIMITIVE_FACTORY(FuncName, TYPE) \
- static inline NodePtr FuncName(const std::string& name, \
- Repetition::type repetition = Repetition::OPTIONAL) { \
- return PrimitiveNode::Make(name, repetition, Type::TYPE); \
- }
-
-PRIMITIVE_FACTORY(Boolean, BOOLEAN);
-PRIMITIVE_FACTORY(Int32, INT32);
-PRIMITIVE_FACTORY(Int64, INT64);
-PRIMITIVE_FACTORY(Int96, INT96);
-PRIMITIVE_FACTORY(Float, FLOAT);
-PRIMITIVE_FACTORY(Double, DOUBLE);
-PRIMITIVE_FACTORY(ByteArray, BYTE_ARRAY);
-
-void PARQUET_EXPORT PrintSchema(const schema::Node* schema, std::ostream& stream,
- int indent_width = 2);
-
-} // namespace schema
-
-// The ColumnDescriptor encapsulates information necessary to interpret
-// primitive column data in the context of a particular schema. We have to
-// examine the node structure of a column's path to the root in the schema tree
-// to be able to reassemble the nested structure from the repetition and
-// definition levels.
-class PARQUET_EXPORT ColumnDescriptor {
- public:
- ColumnDescriptor(const schema::NodePtr& node, int16_t max_definition_level,
- int16_t max_repetition_level,
- const SchemaDescriptor* schema_descr = nullptr);
-
- bool Equals(const ColumnDescriptor& other) const;
-
- int16_t max_definition_level() const { return max_definition_level_; }
-
- int16_t max_repetition_level() const { return max_repetition_level_; }
-
- Type::type physical_type() const { return primitive_node_->physical_type(); }
-
- LogicalType::type logical_type() const { return primitive_node_->logical_type(); }
-
- ColumnOrder column_order() const { return primitive_node_->column_order(); }
-
- SortOrder::type sort_order() const {
- return GetSortOrder(logical_type(), physical_type());
- }
-
- const std::string& name() const { return primitive_node_->name(); }
-
- const std::shared_ptr<schema::ColumnPath> path() const;
-
- const schema::NodePtr& schema_node() const { return node_; }
-
- int type_length() const;
-
- int type_precision() const;
-
- int type_scale() const;
-
- private:
- schema::NodePtr node_;
- const schema::PrimitiveNode* primitive_node_;
-
- int16_t max_definition_level_;
- int16_t max_repetition_level_;
-};
-
-// Container for the converted Parquet schema with a computed information from
-// the schema analysis needed for file reading
-//
-// * Column index to Node
-// * Max repetition / definition levels for each primitive node
-//
-// The ColumnDescriptor objects produced by this class can be used to assist in
-// the reconstruction of fully materialized data structures from the
-// repetition-definition level encoding of nested data
-//
-// TODO(wesm): this object can be recomputed from a Schema
-class PARQUET_EXPORT SchemaDescriptor {
- public:
- SchemaDescriptor() {}
- ~SchemaDescriptor() {}
-
- // Analyze the schema
- void Init(std::unique_ptr<schema::Node> schema);
- void Init(const schema::NodePtr& schema);
-
- const ColumnDescriptor* Column(int i) const;
-
- // Get the index of a column by its dotstring path, or negative value if not found.
- // If several columns share the same dotstring path, it is unspecified which one
- // is returned.
- int ColumnIndex(const std::string& node_path) const;
- // Get the index of a column by its node, or negative value if not found.
- int ColumnIndex(const schema::Node& node) const;
-
- bool Equals(const SchemaDescriptor& other) const;
-
- // The number of physical columns appearing in the file
- int num_columns() const { return static_cast<int>(leaves_.size()); }
-
- const schema::NodePtr& schema_root() const { return schema_; }
-
- const schema::GroupNode* group_node() const { return group_node_; }
-
- // Returns the root (child of the schema root) node of the leaf(column) node
- const schema::Node* GetColumnRoot(int i) const;
-
- const std::string& name() const { return group_node_->name(); }
-
- std::string ToString() const;
-
- void updateColumnOrders(const std::vector<ColumnOrder>& column_orders);
-
- private:
- friend class ColumnDescriptor;
-
- // Root Node
- schema::NodePtr schema_;
- // Root Node
- const schema::GroupNode* group_node_;
-
- void BuildTree(const schema::NodePtr& node, int16_t max_def_level,
- int16_t max_rep_level, const schema::NodePtr& base);
-
- // Result of leaf node / tree analysis
- std::vector<ColumnDescriptor> leaves_;
-
- // Mapping between leaf nodes and root group of leaf (first node
- // below the schema's root group)
- //
- // For example, the leaf `a.b.c.d` would have a link back to `a`
- //
- // -- a <------
- // -- -- b |
- // -- -- -- c |
- // -- -- -- -- d
- std::unordered_map<int, const schema::NodePtr> leaf_to_base_;
-
- // Mapping between ColumnPath DotString to the leaf index
- std::unordered_multimap<std::string, int> leaf_to_idx_;
-};
-
-} // namespace parquet
-
-#endif // PARQUET_SCHEMA_TYPES_H
diff --git a/src/parquet/statistics-test.cc b/src/parquet/statistics-test.cc
deleted file mode 100644
index d2ecede..0000000
--- a/src/parquet/statistics-test.cc
+++ /dev/null
@@ -1,782 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#include <gtest/gtest.h>
-
-#include <algorithm>
-#include <array>
-#include <cstdint>
-#include <cstring>
-#include <memory>
-#include <vector>
-
-#include "parquet/column_reader.h"
-#include "parquet/column_writer.h"
-#include "parquet/file_reader.h"
-#include "parquet/file_writer.h"
-#include "parquet/schema.h"
-#include "parquet/statistics.h"
-#include "parquet/test-specialization.h"
-#include "parquet/test-util.h"
-#include "parquet/thrift.h"
-#include "parquet/types.h"
-#include "parquet/util/memory.h"
-
-using arrow::default_memory_pool;
-using arrow::MemoryPool;
-
-namespace parquet {
-
-using schema::GroupNode;
-using schema::NodePtr;
-using schema::PrimitiveNode;
-
-namespace test {
-
-template <typename TestType>
-class TestRowGroupStatistics : public PrimitiveTypedTest<TestType> {
- public:
- using T = typename TestType::c_type;
- using TypedStats = TypedRowGroupStatistics<TestType>;
-
- std::vector<T> GetDeepCopy(
- const std::vector<T>&); // allocates new memory for FLBA/ByteArray
-
- T* GetValuesPointer(std::vector<T>&);
- void DeepFree(std::vector<T>&);
-
- void TestMinMaxEncode() {
- this->GenerateData(1000);
-
- TypedStats statistics1(this->schema_.Column(0));
- statistics1.Update(this->values_ptr_, this->values_.size(), 0);
- std::string encoded_min = statistics1.EncodeMin();
- std::string encoded_max = statistics1.EncodeMax();
-
- TypedStats statistics2(this->schema_.Column(0), encoded_min, encoded_max,
- this->values_.size(), 0, 0, true);
-
- TypedStats statistics3(this->schema_.Column(0));
- std::vector<uint8_t> valid_bits(
- BitUtil::BytesForBits(static_cast<uint32_t>(this->values_.size())) + 1, 255);
- statistics3.UpdateSpaced(this->values_ptr_, valid_bits.data(), 0,
- this->values_.size(), 0);
- std::string encoded_min_spaced = statistics3.EncodeMin();
- std::string encoded_max_spaced = statistics3.EncodeMax();
-
- ASSERT_EQ(encoded_min, statistics2.EncodeMin());
- ASSERT_EQ(encoded_max, statistics2.EncodeMax());
- ASSERT_EQ(statistics1.min(), statistics2.min());
- ASSERT_EQ(statistics1.max(), statistics2.max());
- ASSERT_EQ(encoded_min_spaced, statistics2.EncodeMin());
- ASSERT_EQ(encoded_max_spaced, statistics2.EncodeMax());
- ASSERT_EQ(statistics3.min(), statistics2.min());
- ASSERT_EQ(statistics3.max(), statistics2.max());
- }
-
- void TestReset() {
- this->GenerateData(1000);
-
- TypedStats statistics(this->schema_.Column(0));
- statistics.Update(this->values_ptr_, this->values_.size(), 0);
- ASSERT_EQ(this->values_.size(), statistics.num_values());
-
- statistics.Reset();
- ASSERT_EQ(0, statistics.null_count());
- ASSERT_EQ(0, statistics.num_values());
- ASSERT_EQ("", statistics.EncodeMin());
- ASSERT_EQ("", statistics.EncodeMax());
- }
-
- void TestMerge() {
- int num_null[2];
- random_numbers(2, 42, 0, 100, num_null);
-
- TypedStats statistics1(this->schema_.Column(0));
- this->GenerateData(1000);
- statistics1.Update(this->values_ptr_, this->values_.size() - num_null[0],
- num_null[0]);
-
- TypedStats statistics2(this->schema_.Column(0));
- this->GenerateData(1000);
- statistics2.Update(this->values_ptr_, this->values_.size() - num_null[1],
- num_null[1]);
-
- TypedStats total(this->schema_.Column(0));
- total.Merge(statistics1);
- total.Merge(statistics2);
-
- ASSERT_EQ(num_null[0] + num_null[1], total.null_count());
- ASSERT_EQ(this->values_.size() * 2 - num_null[0] - num_null[1], total.num_values());
- ASSERT_EQ(total.min(), std::min(statistics1.min(), statistics2.min()));
- ASSERT_EQ(total.max(), std::max(statistics1.max(), statistics2.max()));
- }
-
- void TestFullRoundtrip(int64_t num_values, int64_t null_count) {
- this->GenerateData(num_values);
-
- // compute statistics for the whole batch
- TypedStats expected_stats(this->schema_.Column(0));
- expected_stats.Update(this->values_ptr_, num_values - null_count, null_count);
-
- auto sink = std::make_shared<InMemoryOutputStream>();
- auto gnode = std::static_pointer_cast<GroupNode>(this->node_);
- std::shared_ptr<WriterProperties> writer_properties =
- WriterProperties::Builder().enable_statistics("column")->build();
- auto file_writer = ParquetFileWriter::Open(sink, gnode, writer_properties);
- auto row_group_writer = file_writer->AppendRowGroup();
- auto column_writer =
- static_cast<TypedColumnWriter<TestType>*>(row_group_writer->NextColumn());
-
- // simulate the case when data comes from multiple buffers,
- // in which case special care is necessary for FLBA/ByteArray types
- for (int i = 0; i < 2; i++) {
- int64_t batch_num_values = i ? num_values - num_values / 2 : num_values / 2;
- int64_t batch_null_count = i ? null_count : 0;
- DCHECK(null_count <= num_values); // avoid too much headache
- std::vector<int16_t> definition_levels(batch_null_count, 0);
- definition_levels.insert(definition_levels.end(),
- batch_num_values - batch_null_count, 1);
- auto beg = this->values_.begin() + i * num_values / 2;
- auto end = beg + batch_num_values;
- std::vector<T> batch = GetDeepCopy(std::vector<T>(beg, end));
- T* batch_values_ptr = GetValuesPointer(batch);
- column_writer->WriteBatch(batch_num_values, definition_levels.data(), nullptr,
- batch_values_ptr);
- DeepFree(batch);
- }
- column_writer->Close();
- row_group_writer->Close();
- file_writer->Close();
-
- auto buffer = sink->GetBuffer();
- auto source = std::make_shared<::arrow::io::BufferReader>(buffer);
- auto file_reader = ParquetFileReader::Open(source);
- auto rg_reader = file_reader->RowGroup(0);
- auto column_chunk = rg_reader->metadata()->ColumnChunk(0);
- if (!column_chunk->is_stats_set()) return;
- std::shared_ptr<RowGroupStatistics> stats = column_chunk->statistics();
- // check values after serialization + deserialization
- ASSERT_EQ(null_count, stats->null_count());
- ASSERT_EQ(num_values - null_count, stats->num_values());
- ASSERT_EQ(expected_stats.EncodeMin(), stats->EncodeMin());
- ASSERT_EQ(expected_stats.EncodeMax(), stats->EncodeMax());
- }
-};
-
-template <typename TestType>
-typename TestType::c_type* TestRowGroupStatistics<TestType>::GetValuesPointer(
- std::vector<typename TestType::c_type>& values) {
- return values.data();
-}
-
-template <>
-bool* TestRowGroupStatistics<BooleanType>::GetValuesPointer(std::vector<bool>& values) {
- static std::vector<uint8_t> bool_buffer;
- bool_buffer.clear();
- bool_buffer.resize(values.size());
- std::copy(values.begin(), values.end(), bool_buffer.begin());
- return reinterpret_cast<bool*>(bool_buffer.data());
-}
-
-template <typename TestType>
-typename std::vector<typename TestType::c_type>
-TestRowGroupStatistics<TestType>::GetDeepCopy(
- const std::vector<typename TestType::c_type>& values) {
- return values;
-}
-
-template <>
-std::vector<FLBA> TestRowGroupStatistics<FLBAType>::GetDeepCopy(
- const std::vector<FLBA>& values) {
- std::vector<FLBA> copy;
- MemoryPool* pool = ::arrow::default_memory_pool();
- for (const FLBA& flba : values) {
- uint8_t* ptr;
- PARQUET_THROW_NOT_OK(pool->Allocate(FLBA_LENGTH, &ptr));
- memcpy(ptr, flba.ptr, FLBA_LENGTH);
- copy.emplace_back(ptr);
- }
- return copy;
-}
-
-template <>
-std::vector<ByteArray> TestRowGroupStatistics<ByteArrayType>::GetDeepCopy(
- const std::vector<ByteArray>& values) {
- std::vector<ByteArray> copy;
- MemoryPool* pool = default_memory_pool();
- for (const ByteArray& ba : values) {
- uint8_t* ptr;
- PARQUET_THROW_NOT_OK(pool->Allocate(ba.len, &ptr));
- memcpy(ptr, ba.ptr, ba.len);
- copy.emplace_back(ba.len, ptr);
- }
- return copy;
-}
-
-template <typename TestType>
-void TestRowGroupStatistics<TestType>::DeepFree(
- std::vector<typename TestType::c_type>& values) {}
-
-template <>
-void TestRowGroupStatistics<FLBAType>::DeepFree(std::vector<FLBA>& values) {
- MemoryPool* pool = default_memory_pool();
- for (FLBA& flba : values) {
- auto ptr = const_cast<uint8_t*>(flba.ptr);
- memset(ptr, 0, FLBA_LENGTH);
- pool->Free(ptr, FLBA_LENGTH);
- }
-}
-
-template <>
-void TestRowGroupStatistics<ByteArrayType>::DeepFree(std::vector<ByteArray>& values) {
- MemoryPool* pool = default_memory_pool();
- for (ByteArray& ba : values) {
- auto ptr = const_cast<uint8_t*>(ba.ptr);
- memset(ptr, 0, ba.len);
- pool->Free(ptr, ba.len);
- }
-}
-
-template <>
-void TestRowGroupStatistics<ByteArrayType>::TestMinMaxEncode() {
- this->GenerateData(1000);
- // Test that we encode min max strings correctly
- TypedRowGroupStatistics<ByteArrayType> statistics1(this->schema_.Column(0));
- statistics1.Update(this->values_ptr_, this->values_.size(), 0);
- std::string encoded_min = statistics1.EncodeMin();
- std::string encoded_max = statistics1.EncodeMax();
-
- // encoded is same as unencoded
- ASSERT_EQ(encoded_min,
- std::string((const char*)statistics1.min().ptr, statistics1.min().len));
- ASSERT_EQ(encoded_max,
- std::string((const char*)statistics1.max().ptr, statistics1.max().len));
-
- TypedRowGroupStatistics<ByteArrayType> statistics2(this->schema_.Column(0), encoded_min,
- encoded_max, this->values_.size(), 0,
- 0, true);
-
- ASSERT_EQ(encoded_min, statistics2.EncodeMin());
- ASSERT_EQ(encoded_max, statistics2.EncodeMax());
- ASSERT_EQ(statistics1.min(), statistics2.min());
- ASSERT_EQ(statistics1.max(), statistics2.max());
-}
-
-using TestTypes = ::testing::Types<Int32Type, Int64Type, FloatType, DoubleType,
- ByteArrayType, FLBAType, BooleanType>;
-
-TYPED_TEST_CASE(TestRowGroupStatistics, TestTypes);
-
-TYPED_TEST(TestRowGroupStatistics, MinMaxEncode) {
- this->SetUpSchema(Repetition::REQUIRED);
- ASSERT_NO_FATAL_FAILURE(this->TestMinMaxEncode());
-}
-
-TYPED_TEST(TestRowGroupStatistics, Reset) {
- this->SetUpSchema(Repetition::OPTIONAL);
- ASSERT_NO_FATAL_FAILURE(this->TestReset());
-}
-
-TYPED_TEST(TestRowGroupStatistics, FullRoundtrip) {
- this->SetUpSchema(Repetition::OPTIONAL);
- ASSERT_NO_FATAL_FAILURE(this->TestFullRoundtrip(100, 31));
- ASSERT_NO_FATAL_FAILURE(this->TestFullRoundtrip(1000, 415));
- ASSERT_NO_FATAL_FAILURE(this->TestFullRoundtrip(10000, 926));
-}
-
-template <typename TestType>
-class TestNumericRowGroupStatistics : public TestRowGroupStatistics<TestType> {};
-
-using NumericTypes = ::testing::Types<Int32Type, Int64Type, FloatType, DoubleType>;
-
-TYPED_TEST_CASE(TestNumericRowGroupStatistics, NumericTypes);
-
-TYPED_TEST(TestNumericRowGroupStatistics, Merge) {
- this->SetUpSchema(Repetition::OPTIONAL);
- ASSERT_NO_FATAL_FAILURE(this->TestMerge());
-}
-
-// Statistics are restricted for few types in older parquet version
-TEST(CorruptStatistics, Basics) {
- ApplicationVersion version("parquet-mr version 1.8.0");
- SchemaDescriptor schema;
- schema::NodePtr node;
- std::vector<schema::NodePtr> fields;
- // Test Physical Types
- fields.push_back(schema::PrimitiveNode::Make("col1", Repetition::OPTIONAL, Type::INT32,
- LogicalType::NONE));
- fields.push_back(schema::PrimitiveNode::Make("col2", Repetition::OPTIONAL,
- Type::BYTE_ARRAY, LogicalType::NONE));
- // Test Logical Types
- fields.push_back(schema::PrimitiveNode::Make("col3", Repetition::OPTIONAL, Type::INT32,
- LogicalType::DATE));
- fields.push_back(schema::PrimitiveNode::Make("col4", Repetition::OPTIONAL, Type::INT32,
- LogicalType::UINT_32));
- fields.push_back(schema::PrimitiveNode::Make("col5", Repetition::OPTIONAL,
- Type::FIXED_LEN_BYTE_ARRAY,
- LogicalType::INTERVAL, 12));
- fields.push_back(schema::PrimitiveNode::Make("col6", Repetition::OPTIONAL,
- Type::BYTE_ARRAY, LogicalType::UTF8));
- node = schema::GroupNode::Make("schema", Repetition::REQUIRED, fields);
- schema.Init(node);
-
- format::ColumnChunk col_chunk;
- col_chunk.meta_data.__isset.statistics = true;
- auto column_chunk1 = ColumnChunkMetaData::Make(
- reinterpret_cast<const uint8_t*>(&col_chunk), schema.Column(0), &version);
- ASSERT_TRUE(column_chunk1->is_stats_set());
- auto column_chunk2 = ColumnChunkMetaData::Make(
- reinterpret_cast<const uint8_t*>(&col_chunk), schema.Column(1), &version);
- ASSERT_FALSE(column_chunk2->is_stats_set());
- auto column_chunk3 = ColumnChunkMetaData::Make(
- reinterpret_cast<const uint8_t*>(&col_chunk), schema.Column(2), &version);
- ASSERT_TRUE(column_chunk3->is_stats_set());
- auto column_chunk4 = ColumnChunkMetaData::Make(
- reinterpret_cast<const uint8_t*>(&col_chunk), schema.Column(3), &version);
- ASSERT_FALSE(column_chunk4->is_stats_set());
- auto column_chunk5 = ColumnChunkMetaData::Make(
- reinterpret_cast<const uint8_t*>(&col_chunk), schema.Column(4), &version);
- ASSERT_FALSE(column_chunk5->is_stats_set());
- auto column_chunk6 = ColumnChunkMetaData::Make(
- reinterpret_cast<const uint8_t*>(&col_chunk), schema.Column(5), &version);
- ASSERT_FALSE(column_chunk6->is_stats_set());
-}
-
-// Statistics for all types have no restrictions in newer parquet version
-TEST(CorrectStatistics, Basics) {
- ApplicationVersion version("parquet-cpp version 1.3.0");
- SchemaDescriptor schema;
- schema::NodePtr node;
- std::vector<schema::NodePtr> fields;
- // Test Physical Types
- fields.push_back(schema::PrimitiveNode::Make("col1", Repetition::OPTIONAL, Type::INT32,
- LogicalType::NONE));
- fields.push_back(schema::PrimitiveNode::Make("col2", Repetition::OPTIONAL,
- Type::BYTE_ARRAY, LogicalType::NONE));
- // Test Logical Types
- fields.push_back(schema::PrimitiveNode::Make("col3", Repetition::OPTIONAL, Type::INT32,
- LogicalType::DATE));
- fields.push_back(schema::PrimitiveNode::Make("col4", Repetition::OPTIONAL, Type::INT32,
- LogicalType::UINT_32));
- fields.push_back(schema::PrimitiveNode::Make("col5", Repetition::OPTIONAL,
- Type::FIXED_LEN_BYTE_ARRAY,
- LogicalType::INTERVAL, 12));
- fields.push_back(schema::PrimitiveNode::Make("col6", Repetition::OPTIONAL,
- Type::BYTE_ARRAY, LogicalType::UTF8));
- node = schema::GroupNode::Make("schema", Repetition::REQUIRED, fields);
- schema.Init(node);
-
- format::ColumnChunk col_chunk;
- col_chunk.meta_data.__isset.statistics = true;
- auto column_chunk1 = ColumnChunkMetaData::Make(
- reinterpret_cast<const uint8_t*>(&col_chunk), schema.Column(0), &version);
- ASSERT_TRUE(column_chunk1->is_stats_set());
- auto column_chunk2 = ColumnChunkMetaData::Make(
- reinterpret_cast<const uint8_t*>(&col_chunk), schema.Column(1), &version);
- ASSERT_TRUE(column_chunk2->is_stats_set());
- auto column_chunk3 = ColumnChunkMetaData::Make(
- reinterpret_cast<const uint8_t*>(&col_chunk), schema.Column(2), &version);
- ASSERT_TRUE(column_chunk3->is_stats_set());
- auto column_chunk4 = ColumnChunkMetaData::Make(
- reinterpret_cast<const uint8_t*>(&col_chunk), schema.Column(3), &version);
- ASSERT_TRUE(column_chunk4->is_stats_set());
- auto column_chunk5 = ColumnChunkMetaData::Make(
- reinterpret_cast<const uint8_t*>(&col_chunk), schema.Column(4), &version);
- ASSERT_FALSE(column_chunk5->is_stats_set());
- auto column_chunk6 = ColumnChunkMetaData::Make(
- reinterpret_cast<const uint8_t*>(&col_chunk), schema.Column(5), &version);
- ASSERT_TRUE(column_chunk6->is_stats_set());
-}
-
-// Test SortOrder class
-static const int NUM_VALUES = 10;
-
-template <typename TestType>
-class TestStatistics : public ::testing::Test {
- public:
- typedef typename TestType::c_type T;
-
- void AddNodes(std::string name) {
- fields_.push_back(schema::PrimitiveNode::Make(name, Repetition::REQUIRED,
- TestType::type_num, LogicalType::NONE));
- }
-
- void SetUpSchema() {
- stats_.resize(fields_.size());
- values_.resize(NUM_VALUES);
- schema_ = std::static_pointer_cast<GroupNode>(
- GroupNode::Make("Schema", Repetition::REQUIRED, fields_));
-
- parquet_sink_ = std::make_shared<InMemoryOutputStream>();
- }
-
- void SetValues();
-
- void WriteParquet() {
- // Add writer properties
- parquet::WriterProperties::Builder builder;
- builder.compression(parquet::Compression::SNAPPY);
- builder.created_by("parquet-cpp version 1.3.0");
- std::shared_ptr<parquet::WriterProperties> props = builder.build();
-
- // Create a ParquetFileWriter instance
- auto file_writer = parquet::ParquetFileWriter::Open(parquet_sink_, schema_, props);
-
- // Append a RowGroup with a specific number of rows.
- auto rg_writer = file_writer->AppendRowGroup();
-
- this->SetValues();
-
- // Insert Values
- for (int i = 0; i < static_cast<int>(fields_.size()); i++) {
- auto column_writer =
- static_cast<parquet::TypedColumnWriter<TestType>*>(rg_writer->NextColumn());
- column_writer->WriteBatch(NUM_VALUES, nullptr, nullptr, values_.data());
- }
- }
-
- void VerifyParquetStats() {
- auto pbuffer = parquet_sink_->GetBuffer();
-
- // Create a ParquetReader instance
- std::unique_ptr<parquet::ParquetFileReader> parquet_reader =
- parquet::ParquetFileReader::Open(
- std::make_shared<arrow::io::BufferReader>(pbuffer));
-
- // Get the File MetaData
- std::shared_ptr<parquet::FileMetaData> file_metadata = parquet_reader->metadata();
- std::shared_ptr<parquet::RowGroupMetaData> rg_metadata = file_metadata->RowGroup(0);
- for (int i = 0; i < static_cast<int>(fields_.size()); i++) {
- std::shared_ptr<parquet::ColumnChunkMetaData> cc_metadata =
- rg_metadata->ColumnChunk(i);
- ASSERT_EQ(stats_[i].min(), cc_metadata->statistics()->EncodeMin());
- ASSERT_EQ(stats_[i].max(), cc_metadata->statistics()->EncodeMax());
- }
- }
-
- protected:
- std::vector<T> values_;
- std::vector<uint8_t> values_buf_;
- std::vector<schema::NodePtr> fields_;
- std::shared_ptr<schema::GroupNode> schema_;
- std::shared_ptr<InMemoryOutputStream> parquet_sink_;
- std::vector<EncodedStatistics> stats_;
-};
-
-using CompareTestTypes = ::testing::Types<Int32Type, Int64Type, FloatType, DoubleType,
- ByteArrayType, FLBAType>;
-
-// TYPE::INT32
-template <>
-void TestStatistics<Int32Type>::AddNodes(std::string name) {
- // UINT_32 logical type to set Unsigned Statistics
- fields_.push_back(schema::PrimitiveNode::Make(name, Repetition::REQUIRED, Type::INT32,
- LogicalType::UINT_32));
- // INT_32 logical type to set Signed Statistics
- fields_.push_back(schema::PrimitiveNode::Make(name, Repetition::REQUIRED, Type::INT32,
- LogicalType::INT_32));
-}
-
-template <>
-void TestStatistics<Int32Type>::SetValues() {
- for (int i = 0; i < NUM_VALUES; i++) {
- values_[i] = i - 5; // {-5, -4, -3, -2, -1, 0, 1, 2, 3, 4};
- }
-
- // Write UINT32 min/max values
- stats_[0]
- .set_min(std::string(reinterpret_cast<const char*>(&values_[5]), sizeof(T)))
- .set_max(std::string(reinterpret_cast<const char*>(&values_[4]), sizeof(T)));
-
- // Write INT32 min/max values
- stats_[1]
- .set_min(std::string(reinterpret_cast<const char*>(&values_[0]), sizeof(T)))
- .set_max(std::string(reinterpret_cast<const char*>(&values_[9]), sizeof(T)));
-}
-
-// TYPE::INT64
-template <>
-void TestStatistics<Int64Type>::AddNodes(std::string name) {
- // UINT_64 logical type to set Unsigned Statistics
- fields_.push_back(schema::PrimitiveNode::Make(name, Repetition::REQUIRED, Type::INT64,
- LogicalType::UINT_64));
- // INT_64 logical type to set Signed Statistics
- fields_.push_back(schema::PrimitiveNode::Make(name, Repetition::REQUIRED, Type::INT64,
- LogicalType::INT_64));
-}
-
-template <>
-void TestStatistics<Int64Type>::SetValues() {
- for (int i = 0; i < NUM_VALUES; i++) {
- values_[i] = i - 5; // {-5, -4, -3, -2, -1, 0, 1, 2, 3, 4};
- }
-
- // Write UINT64 min/max values
- stats_[0]
- .set_min(std::string(reinterpret_cast<const char*>(&values_[5]), sizeof(T)))
- .set_max(std::string(reinterpret_cast<const char*>(&values_[4]), sizeof(T)));
-
- // Write INT64 min/max values
- stats_[1]
- .set_min(std::string(reinterpret_cast<const char*>(&values_[0]), sizeof(T)))
- .set_max(std::string(reinterpret_cast<const char*>(&values_[9]), sizeof(T)));
-}
-
-// TYPE::FLOAT
-template <>
-void TestStatistics<FloatType>::SetValues() {
- for (int i = 0; i < NUM_VALUES; i++) {
- values_[i] = static_cast<float>(i) -
- 5; // {-5.0, -4.0, -3.0, -2.0, -1.0, 0.0, 1.0, 2.0, 3.0, 4.0};
- }
-
- // Write Float min/max values
- stats_[0]
- .set_min(std::string(reinterpret_cast<const char*>(&values_[0]), sizeof(T)))
- .set_max(std::string(reinterpret_cast<const char*>(&values_[9]), sizeof(T)));
-}
-
-// TYPE::DOUBLE
-template <>
-void TestStatistics<DoubleType>::SetValues() {
- for (int i = 0; i < NUM_VALUES; i++) {
- values_[i] = static_cast<float>(i) -
- 5; // {-5.0, -4.0, -3.0, -2.0, -1.0, 0.0, 1.0, 2.0, 3.0, 4.0};
- }
-
- // Write Double min/max values
- stats_[0]
- .set_min(std::string(reinterpret_cast<const char*>(&values_[0]), sizeof(T)))
- .set_max(std::string(reinterpret_cast<const char*>(&values_[9]), sizeof(T)));
-}
-
-// TYPE::ByteArray
-template <>
-void TestStatistics<ByteArrayType>::AddNodes(std::string name) {
- // UTF8 logical type to set Unsigned Statistics
- fields_.push_back(schema::PrimitiveNode::Make(name, Repetition::REQUIRED,
- Type::BYTE_ARRAY, LogicalType::UTF8));
-}
-
-template <>
-void TestStatistics<ByteArrayType>::SetValues() {
- int max_byte_array_len = 10;
- size_t nbytes = NUM_VALUES * max_byte_array_len;
- values_buf_.resize(nbytes);
- std::vector<std::string> vals = {u8"c123", u8"b123", u8"a123", u8"d123", u8"e123",
- u8"f123", u8"g123", u8"h123", u8"i123", u8"ü123"};
-
- uint8_t* base = &values_buf_.data()[0];
- for (int i = 0; i < NUM_VALUES; i++) {
- memcpy(base, vals[i].c_str(), vals[i].length());
- values_[i].ptr = base;
- values_[i].len = static_cast<uint32_t>(vals[i].length());
- base += vals[i].length();
- }
-
- // Write String min/max values
- stats_[0]
- .set_min(
- std::string(reinterpret_cast<const char*>(vals[2].c_str()), vals[2].length()))
- .set_max(
- std::string(reinterpret_cast<const char*>(vals[9].c_str()), vals[9].length()));
-}
-
-// TYPE::FLBAArray
-template <>
-void TestStatistics<FLBAType>::AddNodes(std::string name) {
- // FLBA has only Unsigned Statistics
- fields_.push_back(schema::PrimitiveNode::Make(name, Repetition::REQUIRED,
- Type::FIXED_LEN_BYTE_ARRAY,
- LogicalType::NONE, FLBA_LENGTH));
-}
-
-template <>
-void TestStatistics<FLBAType>::SetValues() {
- size_t nbytes = NUM_VALUES * FLBA_LENGTH;
- values_buf_.resize(nbytes);
- char vals[NUM_VALUES][FLBA_LENGTH] = {"b12345", "a12345", "c12345", "d12345", "e12345",
- "f12345", "g12345", "h12345", "z12345", "a12345"};
-
- uint8_t* base = &values_buf_.data()[0];
- for (int i = 0; i < NUM_VALUES; i++) {
- memcpy(base, &vals[i][0], FLBA_LENGTH);
- values_[i].ptr = base;
- base += FLBA_LENGTH;
- }
-
- // Write FLBA min,max values
- stats_[0]
- .set_min(std::string(reinterpret_cast<const char*>(&vals[1][0]), FLBA_LENGTH))
- .set_max(std::string(reinterpret_cast<const char*>(&vals[8][0]), FLBA_LENGTH));
-}
-
-TYPED_TEST_CASE(TestStatistics, CompareTestTypes);
-
-TYPED_TEST(TestStatistics, MinMax) {
- this->AddNodes("Column ");
- this->SetUpSchema();
- this->WriteParquet();
- ASSERT_NO_FATAL_FAILURE(this->VerifyParquetStats());
-}
-
-// Ensure UNKNOWN sort order is handled properly
-using TestStatisticsFLBA = TestStatistics<FLBAType>;
-
-TEST_F(TestStatisticsFLBA, UnknownSortOrder) {
- this->fields_.push_back(schema::PrimitiveNode::Make(
- "Column 0", Repetition::REQUIRED, Type::FIXED_LEN_BYTE_ARRAY, LogicalType::INTERVAL,
- FLBA_LENGTH));
- this->SetUpSchema();
- this->WriteParquet();
-
- auto pbuffer = parquet_sink_->GetBuffer();
- // Create a ParquetReader instance
- std::unique_ptr<parquet::ParquetFileReader> parquet_reader =
- parquet::ParquetFileReader::Open(
- std::make_shared<arrow::io::BufferReader>(pbuffer));
- // Get the File MetaData
- std::shared_ptr<parquet::FileMetaData> file_metadata = parquet_reader->metadata();
- std::shared_ptr<parquet::RowGroupMetaData> rg_metadata = file_metadata->RowGroup(0);
- std::shared_ptr<parquet::ColumnChunkMetaData> cc_metadata = rg_metadata->ColumnChunk(0);
-
- // stats should not be set for UNKNOWN sort order
- ASSERT_FALSE(cc_metadata->is_stats_set());
-}
-
-// PARQUET-1225: Float NaN values may lead to incorrect filtering under certain
-// circumstances
-TEST(TestStatisticsFloatNaN, NaNValues) {
- constexpr int NUM_VALUES = 10;
- NodePtr node = PrimitiveNode::Make("nan_float", Repetition::OPTIONAL, Type::FLOAT);
- ColumnDescriptor descr(node, 1, 1);
- float values[NUM_VALUES] = {std::nanf(""), -4.0f, -3.0f, -2.0f, -1.0f,
- std::nanf(""), 1.0f, 2.0f, 3.0f, std::nanf("")};
- float nan_values[NUM_VALUES];
- for (int i = 0; i < NUM_VALUES; i++) {
- nan_values[i] = std::nanf("");
- }
-
- // Test values
- TypedRowGroupStatistics<FloatType> nan_stats(&descr);
- nan_stats.Update(&values[0], NUM_VALUES, 0);
- float min = nan_stats.min();
- float max = nan_stats.max();
- ASSERT_EQ(min, -4.0f);
- ASSERT_EQ(max, 3.0f);
-
- // Test all NaNs
- TypedRowGroupStatistics<FloatType> all_nan_stats(&descr);
- all_nan_stats.Update(&nan_values[0], NUM_VALUES, 0);
- min = all_nan_stats.min();
- max = all_nan_stats.max();
- ASSERT_TRUE(std::isnan(min));
- ASSERT_TRUE(std::isnan(max));
-
- // Test values followed by all NaNs
- nan_stats.Update(&nan_values[0], NUM_VALUES, 0);
- min = nan_stats.min();
- max = nan_stats.max();
- ASSERT_EQ(min, -4.0f);
- ASSERT_EQ(max, 3.0f);
-
- // Test all NaNs followed by values
- all_nan_stats.Update(&values[0], NUM_VALUES, 0);
- min = all_nan_stats.min();
- max = all_nan_stats.max();
- ASSERT_EQ(min, -4.0f);
- ASSERT_EQ(max, 3.0f);
-
- // Test values followed by all NaNs followed by values
- nan_stats.Update(&values[0], NUM_VALUES, 0);
- min = nan_stats.min();
- max = nan_stats.max();
- ASSERT_EQ(min, -4.0f);
- ASSERT_EQ(max, 3.0f);
-}
-
-// PARQUET-1225: Float NaN values may lead to incorrect filtering under certain
-// circumstances
-TEST(TestStatisticsFloatNaN, NaNValuesSpaced) {
- constexpr int NUM_VALUES = 10;
- NodePtr node = PrimitiveNode::Make("nan_float", Repetition::OPTIONAL, Type::FLOAT);
- ColumnDescriptor descr(node, 1, 1);
- float values[NUM_VALUES] = {std::nanf(""), -4.0f, -3.0f, -2.0f, -1.0f,
- std::nanf(""), 1.0f, 2.0f, 3.0f, std::nanf("")};
- float nan_values[NUM_VALUES];
- for (int i = 0; i < NUM_VALUES; i++) {
- nan_values[i] = std::nanf("");
- }
- std::vector<uint8_t> valid_bits(BitUtil::BytesForBits(NUM_VALUES) + 1, 255);
-
- // Test values
- TypedRowGroupStatistics<FloatType> nan_stats(&descr);
- nan_stats.UpdateSpaced(&values[0], valid_bits.data(), 0, NUM_VALUES, 0);
- float min = nan_stats.min();
- float max = nan_stats.max();
- ASSERT_EQ(min, -4.0f);
- ASSERT_EQ(max, 3.0f);
-
- // Test all NaNs
- TypedRowGroupStatistics<FloatType> all_nan_stats(&descr);
- all_nan_stats.UpdateSpaced(&nan_values[0], valid_bits.data(), 0, NUM_VALUES, 0);
- min = all_nan_stats.min();
- max = all_nan_stats.max();
- ASSERT_TRUE(std::isnan(min));
- ASSERT_TRUE(std::isnan(max));
-
- // Test values followed by all NaNs
- nan_stats.UpdateSpaced(&nan_values[0], valid_bits.data(), 0, NUM_VALUES, 0);
- min = nan_stats.min();
- max = nan_stats.max();
- ASSERT_EQ(min, -4.0f);
- ASSERT_EQ(max, 3.0f);
-
- // Test all NaNs followed by values
- all_nan_stats.UpdateSpaced(&values[0], valid_bits.data(), 0, NUM_VALUES, 0);
- min = all_nan_stats.min();
- max = all_nan_stats.max();
- ASSERT_EQ(min, -4.0f);
- ASSERT_EQ(max, 3.0f);
-
- // Test values followed by all NaNs followed by values
- nan_stats.UpdateSpaced(&values[0], valid_bits.data(), 0, NUM_VALUES, 0);
- min = nan_stats.min();
- max = nan_stats.max();
- ASSERT_EQ(min, -4.0f);
- ASSERT_EQ(max, 3.0f);
-}
-
-// NaN double values may lead to incorrect filtering under certain circumstances
-TEST(TestStatisticsDoubleNaN, NaNValues) {
- constexpr int NUM_VALUES = 10;
- NodePtr node = PrimitiveNode::Make("nan_double", Repetition::OPTIONAL, Type::DOUBLE);
- ColumnDescriptor descr(node, 1, 1);
- TypedRowGroupStatistics<DoubleType> nan_stats(&descr);
- double values[NUM_VALUES] = {std::nan(""), std::nan(""), -3.0, -2.0, -1.0,
- 0.0, 1.0, 2.0, 3.0, 4.0};
- double* values_ptr = &values[0];
- nan_stats.Update(values_ptr, NUM_VALUES, 0);
- double min = nan_stats.min();
- double max = nan_stats.max();
-
- ASSERT_EQ(min, -3.0);
- ASSERT_EQ(max, 4.0);
-}
-} // namespace test
-} // namespace parquet
diff --git a/src/parquet/statistics.cc b/src/parquet/statistics.cc
deleted file mode 100644
index ea7f783..0000000
--- a/src/parquet/statistics.cc
+++ /dev/null
@@ -1,334 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#include <algorithm>
-#include <cstring>
-#include <type_traits>
-
-#include "parquet/encoding-internal.h"
-#include "parquet/exception.h"
-#include "parquet/statistics.h"
-#include "parquet/util/memory.h"
-
-using arrow::default_memory_pool;
-using arrow::MemoryPool;
-
-namespace parquet {
-
-template <typename DType>
-TypedRowGroupStatistics<DType>::TypedRowGroupStatistics(const ColumnDescriptor* schema,
- MemoryPool* pool)
- : pool_(pool),
- min_buffer_(AllocateBuffer(pool_, 0)),
- max_buffer_(AllocateBuffer(pool_, 0)) {
- SetDescr(schema);
- Reset();
-}
-
-template <typename DType>
-TypedRowGroupStatistics<DType>::TypedRowGroupStatistics(const typename DType::c_type& min,
- const typename DType::c_type& max,
- int64_t num_values,
- int64_t null_count,
- int64_t distinct_count)
- : pool_(default_memory_pool()),
- min_buffer_(AllocateBuffer(pool_, 0)),
- max_buffer_(AllocateBuffer(pool_, 0)) {
- IncrementNumValues(num_values);
- IncrementNullCount(null_count);
- IncrementDistinctCount(distinct_count);
-
- Copy(min, &min_, min_buffer_.get());
- Copy(max, &max_, max_buffer_.get());
- has_min_max_ = true;
-}
-
-template <typename DType>
-TypedRowGroupStatistics<DType>::TypedRowGroupStatistics(
- const ColumnDescriptor* schema, const std::string& encoded_min,
- const std::string& encoded_max, int64_t num_values, int64_t null_count,
- int64_t distinct_count, bool has_min_max, MemoryPool* pool)
- : pool_(pool),
- min_buffer_(AllocateBuffer(pool_, 0)),
- max_buffer_(AllocateBuffer(pool_, 0)) {
- IncrementNumValues(num_values);
- IncrementNullCount(null_count);
- IncrementDistinctCount(distinct_count);
-
- SetDescr(schema);
-
- if (!encoded_min.empty()) {
- PlainDecode(encoded_min, &min_);
- }
- if (!encoded_max.empty()) {
- PlainDecode(encoded_max, &max_);
- }
- has_min_max_ = has_min_max;
-}
-
-template <typename DType>
-bool TypedRowGroupStatistics<DType>::HasMinMax() const {
- return has_min_max_;
-}
-
-template <typename DType>
-void TypedRowGroupStatistics<DType>::SetComparator() {
- comparator_ =
- std::static_pointer_cast<CompareDefault<DType> >(Comparator::Make(descr_));
-}
-
-template <typename DType>
-void TypedRowGroupStatistics<DType>::Reset() {
- ResetCounts();
- has_min_max_ = false;
-}
-
-template <typename DType>
-inline void TypedRowGroupStatistics<DType>::SetMinMax(const T& min, const T& max) {
- if (!has_min_max_) {
- has_min_max_ = true;
- Copy(min, &min_, min_buffer_.get());
- Copy(max, &max_, max_buffer_.get());
- } else {
- Copy(std::min(min_, min, std::ref(*(this->comparator_))), &min_, min_buffer_.get());
- Copy(std::max(max_, max, std::ref(*(this->comparator_))), &max_, max_buffer_.get());
- }
-}
-
-template <typename T, typename Enable = void>
-struct StatsHelper {
- inline int64_t GetValueBeginOffset(const T* values, int64_t count) { return 0; }
-
- inline int64_t GetValueEndOffset(const T* values, int64_t count) { return count; }
-
- inline bool IsNaN(const T value) { return false; }
-};
-
-template <typename T>
-struct StatsHelper<T, typename std::enable_if<std::is_floating_point<T>::value>::type> {
- inline int64_t GetValueBeginOffset(const T* values, int64_t count) {
- // Skip NaNs
- for (int64_t i = 0; i < count; i++) {
- if (!std::isnan(values[i])) {
- return i;
- }
- }
- return count;
- }
-
- inline int64_t GetValueEndOffset(const T* values, int64_t count) {
- // Skip NaNs
- for (int64_t i = (count - 1); i >= 0; i--) {
- if (!std::isnan(values[i])) {
- return (i + 1);
- }
- }
- return 0;
- }
-
- inline bool IsNaN(const T value) { return std::isnan(value); }
-};
-
-template <typename T>
-void SetNaN(T* value) {
- // no-op
-}
-
-template <>
-void SetNaN<float>(float* value) {
- *value = std::nanf("");
-}
-
-template <>
-void SetNaN<double>(double* value) {
- *value = std::nan("");
-}
-
-template <typename DType>
-void TypedRowGroupStatistics<DType>::Update(const T* values, int64_t num_not_null,
- int64_t num_null) {
- DCHECK(num_not_null >= 0);
- DCHECK(num_null >= 0);
-
- IncrementNullCount(num_null);
- IncrementNumValues(num_not_null);
- // TODO: support distinct count?
- if (num_not_null == 0) return;
-
- // PARQUET-1225: Handle NaNs
- // The problem arises only if the starting/ending value(s)
- // of the values-buffer contain NaN
- StatsHelper<T> helper;
- int64_t begin_offset = helper.GetValueBeginOffset(values, num_not_null);
- int64_t end_offset = helper.GetValueEndOffset(values, num_not_null);
-
- // All values are NaN
- if (end_offset < begin_offset) {
- // Set min/max to NaNs in this case.
- // Don't set has_min_max flag since
- // these values must be over-written by valid stats later
- if (!has_min_max_) {
- SetNaN(&min_);
- SetNaN(&max_);
- }
- return;
- }
-
- auto batch_minmax = std::minmax_element(values + begin_offset, values + end_offset,
- std::ref(*(this->comparator_)));
-
- SetMinMax(*batch_minmax.first, *batch_minmax.second);
-}
-
-template <typename DType>
-void TypedRowGroupStatistics<DType>::UpdateSpaced(const T* values,
- const uint8_t* valid_bits,
- int64_t valid_bits_offset,
- int64_t num_not_null,
- int64_t num_null) {
- DCHECK(num_not_null >= 0);
- DCHECK(num_null >= 0);
-
- IncrementNullCount(num_null);
- IncrementNumValues(num_not_null);
- // TODO: support distinct count?
- if (num_not_null == 0) return;
-
- // Find first valid entry and use that for min/max
- // As (num_not_null != 0) there must be one
- int64_t length = num_null + num_not_null;
- int64_t i = 0;
- ::arrow::internal::BitmapReader valid_bits_reader(valid_bits, valid_bits_offset,
- length);
- StatsHelper<T> helper;
- for (; i < length; i++) {
- // PARQUET-1225: Handle NaNs
- if (valid_bits_reader.IsSet() && !helper.IsNaN(values[i])) {
- break;
- }
- valid_bits_reader.Next();
- }
-
- // All are NaNs and stats are not set yet
- if ((i == length) && helper.IsNaN(values[i - 1])) {
- // Don't set has_min_max flag since
- // these values must be over-written by valid stats later
- if (!has_min_max_) {
- SetNaN(&min_);
- SetNaN(&max_);
- }
- return;
- }
-
- T min = values[i];
- T max = values[i];
- for (; i < length; i++) {
- if (valid_bits_reader.IsSet()) {
- if ((std::ref(*(this->comparator_)))(values[i], min)) {
- min = values[i];
- } else if ((std::ref(*(this->comparator_)))(max, values[i])) {
- max = values[i];
- }
- }
- valid_bits_reader.Next();
- }
-
- SetMinMax(min, max);
-}
-
-template <typename DType>
-const typename DType::c_type& TypedRowGroupStatistics<DType>::min() const {
- return min_;
-}
-
-template <typename DType>
-const typename DType::c_type& TypedRowGroupStatistics<DType>::max() const {
- return max_;
-}
-
-template <typename DType>
-void TypedRowGroupStatistics<DType>::Merge(const TypedRowGroupStatistics<DType>& other) {
- this->MergeCounts(other);
-
- if (!other.HasMinMax()) return;
-
- SetMinMax(other.min_, other.max_);
-}
-
-template <typename DType>
-std::string TypedRowGroupStatistics<DType>::EncodeMin() {
- std::string s;
- if (HasMinMax()) this->PlainEncode(min_, &s);
- return s;
-}
-
-template <typename DType>
-std::string TypedRowGroupStatistics<DType>::EncodeMax() {
- std::string s;
- if (HasMinMax()) this->PlainEncode(max_, &s);
- return s;
-}
-
-template <typename DType>
-EncodedStatistics TypedRowGroupStatistics<DType>::Encode() {
- EncodedStatistics s;
- if (HasMinMax()) {
- s.set_min(this->EncodeMin());
- s.set_max(this->EncodeMax());
- }
- s.set_null_count(this->null_count());
- return s;
-}
-
-template <typename DType>
-void TypedRowGroupStatistics<DType>::PlainEncode(const T& src, std::string* dst) {
- PlainEncoder<DType> encoder(descr(), pool_);
- encoder.Put(&src, 1);
- auto buffer = encoder.FlushValues();
- auto ptr = reinterpret_cast<const char*>(buffer->data());
- dst->assign(ptr, buffer->size());
-}
-
-template <typename DType>
-void TypedRowGroupStatistics<DType>::PlainDecode(const std::string& src, T* dst) {
- PlainDecoder<DType> decoder(descr());
- decoder.SetData(1, reinterpret_cast<const uint8_t*>(src.c_str()),
- static_cast<int>(src.size()));
- decoder.Decode(dst, 1);
-}
-
-template <>
-void TypedRowGroupStatistics<ByteArrayType>::PlainEncode(const T& src, std::string* dst) {
- dst->assign(reinterpret_cast<const char*>(src.ptr), src.len);
-}
-
-template <>
-void TypedRowGroupStatistics<ByteArrayType>::PlainDecode(const std::string& src, T* dst) {
- dst->len = static_cast<uint32_t>(src.size());
- dst->ptr = reinterpret_cast<const uint8_t*>(src.c_str());
-}
-
-template class PARQUET_TEMPLATE_EXPORT TypedRowGroupStatistics<BooleanType>;
-template class PARQUET_TEMPLATE_EXPORT TypedRowGroupStatistics<Int32Type>;
-template class PARQUET_TEMPLATE_EXPORT TypedRowGroupStatistics<Int64Type>;
-template class PARQUET_TEMPLATE_EXPORT TypedRowGroupStatistics<Int96Type>;
-template class PARQUET_TEMPLATE_EXPORT TypedRowGroupStatistics<FloatType>;
-template class PARQUET_TEMPLATE_EXPORT TypedRowGroupStatistics<DoubleType>;
-template class PARQUET_TEMPLATE_EXPORT TypedRowGroupStatistics<ByteArrayType>;
-template class PARQUET_TEMPLATE_EXPORT TypedRowGroupStatistics<FLBAType>;
-
-} // namespace parquet
diff --git a/src/parquet/statistics.h b/src/parquet/statistics.h
deleted file mode 100644
index d1c4d16..0000000
--- a/src/parquet/statistics.h
+++ /dev/null
@@ -1,249 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#ifndef PARQUET_COLUMN_STATISTICS_H
-#define PARQUET_COLUMN_STATISTICS_H
-
-#include <algorithm>
-#include <cstdint>
-#include <memory>
-#include <string>
-
-#include "parquet/schema.h"
-#include "parquet/types.h"
-#include "parquet/util/comparison.h"
-#include "parquet/util/memory.h"
-#include "parquet/util/visibility.h"
-
-namespace parquet {
-
-class PARQUET_EXPORT EncodedStatistics {
- std::shared_ptr<std::string> max_, min_;
-
- public:
- EncodedStatistics()
- : max_(std::make_shared<std::string>()), min_(std::make_shared<std::string>()) {}
-
- const std::string& max() const { return *max_; }
- const std::string& min() const { return *min_; }
-
- int64_t null_count = 0;
- int64_t distinct_count = 0;
-
- bool has_min = false;
- bool has_max = false;
- bool has_null_count = false;
- bool has_distinct_count = false;
-
- inline bool is_set() const {
- return has_min || has_max || has_null_count || has_distinct_count;
- }
-
- // larger of the max_ and min_ stat values
- inline size_t max_stat_length() { return std::max(max_->length(), min_->length()); }
-
- inline EncodedStatistics& set_max(const std::string& value) {
- *max_ = value;
- has_max = true;
- return *this;
- }
-
- inline EncodedStatistics& set_min(const std::string& value) {
- *min_ = value;
- has_min = true;
- return *this;
- }
-
- inline EncodedStatistics& set_null_count(int64_t value) {
- null_count = value;
- has_null_count = true;
- return *this;
- }
-
- inline EncodedStatistics& set_distinct_count(int64_t value) {
- distinct_count = value;
- has_distinct_count = true;
- return *this;
- }
-};
-
-template <typename DType>
-class PARQUET_EXPORT TypedRowGroupStatistics;
-
-class PARQUET_EXPORT RowGroupStatistics
- : public std::enable_shared_from_this<RowGroupStatistics> {
- public:
- int64_t null_count() const { return statistics_.null_count; }
- int64_t distinct_count() const { return statistics_.distinct_count; }
- int64_t num_values() const { return num_values_; }
-
- virtual bool HasMinMax() const = 0;
- virtual void Reset() = 0;
-
- // Plain-encoded minimum value
- virtual std::string EncodeMin() = 0;
-
- // Plain-encoded maximum value
- virtual std::string EncodeMax() = 0;
-
- virtual EncodedStatistics Encode() = 0;
-
- // Set the Corresponding Comparator
- virtual void SetComparator() = 0;
-
- virtual ~RowGroupStatistics() {}
-
- Type::type physical_type() const { return descr_->physical_type(); }
-
- protected:
- const ColumnDescriptor* descr() const { return descr_; }
- void SetDescr(const ColumnDescriptor* schema) {
- descr_ = schema;
- SetComparator();
- }
-
- void IncrementNullCount(int64_t n) { statistics_.null_count += n; }
-
- void IncrementNumValues(int64_t n) { num_values_ += n; }
-
- void IncrementDistinctCount(int64_t n) { statistics_.distinct_count += n; }
-
- void MergeCounts(const RowGroupStatistics& other) {
- this->statistics_.null_count += other.statistics_.null_count;
- this->statistics_.distinct_count += other.statistics_.distinct_count;
- this->num_values_ += other.num_values_;
- }
-
- void ResetCounts() {
- this->statistics_.null_count = 0;
- this->statistics_.distinct_count = 0;
- this->num_values_ = 0;
- }
-
- const ColumnDescriptor* descr_ = nullptr;
- int64_t num_values_ = 0;
- EncodedStatistics statistics_;
-};
-
-template <typename DType>
-class TypedRowGroupStatistics : public RowGroupStatistics {
- public:
- using T = typename DType::c_type;
-
- TypedRowGroupStatistics(const ColumnDescriptor* schema,
- ::arrow::MemoryPool* pool = ::arrow::default_memory_pool());
-
- TypedRowGroupStatistics(const T& min, const T& max, int64_t num_values,
- int64_t null_count, int64_t distinct_count);
-
- TypedRowGroupStatistics(const ColumnDescriptor* schema, const std::string& encoded_min,
- const std::string& encoded_max, int64_t num_values,
- int64_t null_count, int64_t distinct_count, bool has_min_max,
- ::arrow::MemoryPool* pool = ::arrow::default_memory_pool());
-
- bool HasMinMax() const override;
- void Reset() override;
- void SetComparator() override;
- void Merge(const TypedRowGroupStatistics<DType>& other);
-
- void Update(const T* values, int64_t num_not_null, int64_t num_null);
- void UpdateSpaced(const T* values, const uint8_t* valid_bits, int64_t valid_bits_spaced,
- int64_t num_not_null, int64_t num_null);
- void SetMinMax(const T& min, const T& max);
-
- const T& min() const;
- const T& max() const;
-
- std::string EncodeMin() override;
- std::string EncodeMax() override;
- EncodedStatistics Encode() override;
-
- private:
- bool has_min_max_ = false;
- T min_;
- T max_;
- ::arrow::MemoryPool* pool_;
- std::shared_ptr<CompareDefault<DType> > comparator_;
-
- void PlainEncode(const T& src, std::string* dst);
- void PlainDecode(const std::string& src, T* dst);
- void Copy(const T& src, T* dst, ResizableBuffer* buffer);
-
- std::shared_ptr<ResizableBuffer> min_buffer_, max_buffer_;
-};
-
-template <typename DType>
-inline void TypedRowGroupStatistics<DType>::Copy(const T& src, T* dst, ResizableBuffer*) {
- *dst = src;
-}
-
-template <>
-inline void TypedRowGroupStatistics<FLBAType>::Copy(const FLBA& src, FLBA* dst,
- ResizableBuffer* buffer) {
- if (dst->ptr == src.ptr) return;
- uint32_t len = descr_->type_length();
- PARQUET_THROW_NOT_OK(buffer->Resize(len, false));
- std::memcpy(buffer->mutable_data(), src.ptr, len);
- *dst = FLBA(buffer->data());
-}
-
-template <>
-inline void TypedRowGroupStatistics<ByteArrayType>::Copy(const ByteArray& src,
- ByteArray* dst,
- ResizableBuffer* buffer) {
- if (dst->ptr == src.ptr) return;
- PARQUET_THROW_NOT_OK(buffer->Resize(src.len, false));
- std::memcpy(buffer->mutable_data(), src.ptr, src.len);
- *dst = ByteArray(src.len, buffer->data());
-}
-
-template <>
-void TypedRowGroupStatistics<ByteArrayType>::PlainEncode(const T& src, std::string* dst);
-
-template <>
-void TypedRowGroupStatistics<ByteArrayType>::PlainDecode(const std::string& src, T* dst);
-
-typedef TypedRowGroupStatistics<BooleanType> BoolStatistics;
-typedef TypedRowGroupStatistics<Int32Type> Int32Statistics;
-typedef TypedRowGroupStatistics<Int64Type> Int64Statistics;
-typedef TypedRowGroupStatistics<Int96Type> Int96Statistics;
-typedef TypedRowGroupStatistics<FloatType> FloatStatistics;
-typedef TypedRowGroupStatistics<DoubleType> DoubleStatistics;
-typedef TypedRowGroupStatistics<ByteArrayType> ByteArrayStatistics;
-typedef TypedRowGroupStatistics<FLBAType> FLBAStatistics;
-
-#if defined(__GNUC__) && !defined(__clang__)
-#pragma GCC diagnostic push
-#pragma GCC diagnostic ignored "-Wattributes"
-#endif
-
-PARQUET_EXTERN_TEMPLATE TypedRowGroupStatistics<BooleanType>;
-PARQUET_EXTERN_TEMPLATE TypedRowGroupStatistics<Int32Type>;
-PARQUET_EXTERN_TEMPLATE TypedRowGroupStatistics<Int64Type>;
-PARQUET_EXTERN_TEMPLATE TypedRowGroupStatistics<Int96Type>;
-PARQUET_EXTERN_TEMPLATE TypedRowGroupStatistics<FloatType>;
-PARQUET_EXTERN_TEMPLATE TypedRowGroupStatistics<DoubleType>;
-PARQUET_EXTERN_TEMPLATE TypedRowGroupStatistics<ByteArrayType>;
-PARQUET_EXTERN_TEMPLATE TypedRowGroupStatistics<FLBAType>;
-
-#if defined(__GNUC__) && !defined(__clang__)
-#pragma GCC diagnostic pop
-#endif
-
-} // namespace parquet
-
-#endif // PARQUET_COLUMN_STATISTICS_H
diff --git a/src/parquet/symbols.map b/src/parquet/symbols.map
deleted file mode 100644
index e00751e..0000000
--- a/src/parquet/symbols.map
+++ /dev/null
@@ -1,38 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-{
- # Symbols marked as 'local' are not exported by the DSO and thus may not
- # be used by client applications.
- local:
- # devtoolset / static-libstdc++ symbols
- __cxa_*;
-
- extern "C++" {
- # boost
- boost::*;
-
- # thrift
- apache::thrift::*;
-
- # devtoolset or -static-libstdc++ - the Red Hat devtoolset statically
- # links c++11 symbols into binaries so that the result may be executed on
- # a system with an older libstdc++ which doesn't include the necessary
- # c++11 symbols.
- std::*;
- };
-};
diff --git a/src/parquet/test-specialization.h b/src/parquet/test-specialization.h
deleted file mode 100644
index 3d88cfc..0000000
--- a/src/parquet/test-specialization.h
+++ /dev/null
@@ -1,183 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-// This module defines an abstract interface for iterating through pages in a
-// Parquet column chunk within a row group. It could be extended in the future
-// to iterate through all data pages in all chunks in a file.
-
-#ifndef PARQUET_COLUMN_TEST_SPECIALIZATION_H
-#define PARQUET_COLUMN_TEST_SPECIALIZATION_H
-
-#include <algorithm>
-#include <limits>
-#include <sstream>
-#include <string>
-#include <vector>
-
-#include "parquet/test-util.h"
-
-namespace parquet {
-
-namespace test {
-
-template <>
-void inline InitValues<bool>(int num_values, vector<bool>& values,
- vector<uint8_t>& buffer) {
- values = flip_coins(num_values, 0);
-}
-
-template <>
-void inline InitValues<ByteArray>(int num_values, vector<ByteArray>& values,
- vector<uint8_t>& buffer) {
- int max_byte_array_len = 12;
- int num_bytes = static_cast<int>(max_byte_array_len + sizeof(uint32_t));
- size_t nbytes = num_values * num_bytes;
- buffer.resize(nbytes);
- random_byte_array(num_values, 0, buffer.data(), values.data(), max_byte_array_len);
-}
-
-void inline InitWideByteArrayValues(int num_values, vector<ByteArray>& values,
- vector<uint8_t>& buffer, int min_len, int max_len) {
- int num_bytes = static_cast<int>(max_len + sizeof(uint32_t));
- size_t nbytes = num_values * num_bytes;
- buffer.resize(nbytes);
- random_byte_array(num_values, 0, buffer.data(), values.data(), min_len, max_len);
-}
-
-template <>
-void inline InitValues<FLBA>(int num_values, vector<FLBA>& values,
- vector<uint8_t>& buffer) {
- size_t nbytes = num_values * FLBA_LENGTH;
- buffer.resize(nbytes);
- random_fixed_byte_array(num_values, 0, buffer.data(), FLBA_LENGTH, values.data());
-}
-
-template <>
-void inline InitValues<Int96>(int num_values, vector<Int96>& values,
- vector<uint8_t>& buffer) {
- random_Int96_numbers(num_values, 0, std::numeric_limits<int32_t>::min(),
- std::numeric_limits<int32_t>::max(), values.data());
-}
-
-inline std::string TestColumnName(int i) {
- std::stringstream col_name;
- col_name << "column_" << i;
- return col_name.str();
-}
-
-// This class lives here because of its dependency on the InitValues specializations.
-template <typename TestType>
-class PrimitiveTypedTest : public ::testing::Test {
- public:
- typedef typename TestType::c_type T;
-
- void SetUpSchema(Repetition::type repetition, int num_columns = 1) {
- std::vector<schema::NodePtr> fields;
-
- for (int i = 0; i < num_columns; ++i) {
- std::string name = TestColumnName(i);
- fields.push_back(schema::PrimitiveNode::Make(name, repetition, TestType::type_num,
- LogicalType::NONE, FLBA_LENGTH));
- }
- node_ = schema::GroupNode::Make("schema", Repetition::REQUIRED, fields);
- schema_.Init(node_);
- }
-
- void GenerateData(int64_t num_values);
- void SetupValuesOut(int64_t num_values);
- void SyncValuesOut();
-
- protected:
- schema::NodePtr node_;
- SchemaDescriptor schema_;
-
- // Input buffers
- std::vector<T> values_;
-
- std::vector<int16_t> def_levels_;
-
- std::vector<uint8_t> buffer_;
- // Pointer to the values, needed as we cannot use vector<bool>::data()
- T* values_ptr_;
- std::vector<uint8_t> bool_buffer_;
-
- // Output buffers
- std::vector<T> values_out_;
- std::vector<uint8_t> bool_buffer_out_;
- T* values_out_ptr_;
-};
-
-template <typename TestType>
-void PrimitiveTypedTest<TestType>::SyncValuesOut() {}
-
-template <>
-void PrimitiveTypedTest<BooleanType>::SyncValuesOut() {
- std::vector<uint8_t>::const_iterator source_iterator = bool_buffer_out_.begin();
- std::vector<T>::iterator destination_iterator = values_out_.begin();
- while (source_iterator != bool_buffer_out_.end()) {
- *destination_iterator++ = *source_iterator++ != 0;
- }
-}
-
-template <typename TestType>
-void PrimitiveTypedTest<TestType>::SetupValuesOut(int64_t num_values) {
- values_out_.clear();
- values_out_.resize(num_values);
- values_out_ptr_ = values_out_.data();
-}
-
-template <>
-void PrimitiveTypedTest<BooleanType>::SetupValuesOut(int64_t num_values) {
- values_out_.clear();
- values_out_.resize(num_values);
-
- bool_buffer_out_.clear();
- bool_buffer_out_.resize(num_values);
- // Write once to all values so we can copy it without getting Valgrind errors
- // about uninitialised values.
- std::fill(bool_buffer_out_.begin(), bool_buffer_out_.end(), true);
- values_out_ptr_ = reinterpret_cast<bool*>(bool_buffer_out_.data());
-}
-
-template <typename TestType>
-void PrimitiveTypedTest<TestType>::GenerateData(int64_t num_values) {
- def_levels_.resize(num_values);
- values_.resize(num_values);
-
- InitValues<T>(static_cast<int>(num_values), values_, buffer_);
- values_ptr_ = values_.data();
-
- std::fill(def_levels_.begin(), def_levels_.end(), 1);
-}
-
-template <>
-void PrimitiveTypedTest<BooleanType>::GenerateData(int64_t num_values) {
- def_levels_.resize(num_values);
- values_.resize(num_values);
-
- InitValues<T>(static_cast<int>(num_values), values_, buffer_);
- bool_buffer_.resize(num_values);
- std::copy(values_.begin(), values_.end(), bool_buffer_.begin());
- values_ptr_ = reinterpret_cast<bool*>(bool_buffer_.data());
-
- std::fill(def_levels_.begin(), def_levels_.end(), 1);
-}
-} // namespace test
-
-} // namespace parquet
-
-#endif // PARQUET_COLUMN_TEST_SPECIALIZATION_H
diff --git a/src/parquet/test-util.h b/src/parquet/test-util.h
deleted file mode 100644
index 3e74398..0000000
--- a/src/parquet/test-util.h
+++ /dev/null
@@ -1,447 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-// This module defines an abstract interface for iterating through pages in a
-// Parquet column chunk within a row group. It could be extended in the future
-// to iterate through all data pages in all chunks in a file.
-
-#ifndef PARQUET_COLUMN_TEST_UTIL_H
-#define PARQUET_COLUMN_TEST_UTIL_H
-
-#include <algorithm>
-#include <limits>
-#include <memory>
-#include <string>
-#include <vector>
-
-#include <gtest/gtest.h>
-
-#include "parquet/column_page.h"
-#include "parquet/column_reader.h"
-#include "parquet/column_writer.h"
-#include "parquet/encoding-internal.h"
-#include "parquet/util/memory.h"
-#include "parquet/util/test-common.h"
-
-using std::shared_ptr;
-using std::vector;
-
-namespace parquet {
-
-static constexpr int FLBA_LENGTH = 12;
-
-bool operator==(const FixedLenByteArray& a, const FixedLenByteArray& b) {
- return 0 == memcmp(a.ptr, b.ptr, FLBA_LENGTH);
-}
-
-namespace test {
-
-template <typename T>
-static void InitValues(int num_values, vector<T>& values, vector<uint8_t>& buffer) {
- random_numbers(num_values, 0, std::numeric_limits<T>::min(),
- std::numeric_limits<T>::max(), values.data());
-}
-
-template <typename T>
-static void InitDictValues(int num_values, int num_dicts, vector<T>& values,
- vector<uint8_t>& buffer) {
- int repeat_factor = num_values / num_dicts;
- InitValues<T>(num_dicts, values, buffer);
- // add some repeated values
- for (int j = 1; j < repeat_factor; ++j) {
- for (int i = 0; i < num_dicts; ++i) {
- std::memcpy(&values[num_dicts * j + i], &values[i], sizeof(T));
- }
- }
- // computed only dict_per_page * repeat_factor - 1 values < num_values
- // compute remaining
- for (int i = num_dicts * repeat_factor; i < num_values; ++i) {
- std::memcpy(&values[i], &values[i - num_dicts * repeat_factor], sizeof(T));
- }
-}
-
-class MockPageReader : public PageReader {
- public:
- explicit MockPageReader(const vector<shared_ptr<Page>>& pages)
- : pages_(pages), page_index_(0) {}
-
- shared_ptr<Page> NextPage() override {
- if (page_index_ == static_cast<int>(pages_.size())) {
- // EOS to consumer
- return shared_ptr<Page>(nullptr);
- }
- return pages_[page_index_++];
- }
-
- // No-op
- void set_max_page_header_size(uint32_t size) override {}
-
- private:
- vector<shared_ptr<Page>> pages_;
- int page_index_;
-};
-
-// TODO(wesm): this is only used for testing for now. Refactor to form part of
-// primary file write path
-template <typename Type>
-class DataPageBuilder {
- public:
- typedef typename Type::c_type T;
-
- // This class writes data and metadata to the passed inputs
- explicit DataPageBuilder(InMemoryOutputStream* sink)
- : sink_(sink),
- num_values_(0),
- encoding_(Encoding::PLAIN),
- definition_level_encoding_(Encoding::RLE),
- repetition_level_encoding_(Encoding::RLE),
- have_def_levels_(false),
- have_rep_levels_(false),
- have_values_(false) {}
-
- void AppendDefLevels(const vector<int16_t>& levels, int16_t max_level,
- Encoding::type encoding = Encoding::RLE) {
- AppendLevels(levels, max_level, encoding);
-
- num_values_ = std::max(static_cast<int32_t>(levels.size()), num_values_);
- definition_level_encoding_ = encoding;
- have_def_levels_ = true;
- }
-
- void AppendRepLevels(const vector<int16_t>& levels, int16_t max_level,
- Encoding::type encoding = Encoding::RLE) {
- AppendLevels(levels, max_level, encoding);
-
- num_values_ = std::max(static_cast<int32_t>(levels.size()), num_values_);
- repetition_level_encoding_ = encoding;
- have_rep_levels_ = true;
- }
-
- void AppendValues(const ColumnDescriptor* d, const vector<T>& values,
- Encoding::type encoding = Encoding::PLAIN) {
- PlainEncoder<Type> encoder(d);
- encoder.Put(&values[0], static_cast<int>(values.size()));
- std::shared_ptr<Buffer> values_sink = encoder.FlushValues();
- sink_->Write(values_sink->data(), values_sink->size());
-
- num_values_ = std::max(static_cast<int32_t>(values.size()), num_values_);
- encoding_ = encoding;
- have_values_ = true;
- }
-
- int32_t num_values() const { return num_values_; }
-
- Encoding::type encoding() const { return encoding_; }
-
- Encoding::type rep_level_encoding() const { return repetition_level_encoding_; }
-
- Encoding::type def_level_encoding() const { return definition_level_encoding_; }
-
- private:
- InMemoryOutputStream* sink_;
-
- int32_t num_values_;
- Encoding::type encoding_;
- Encoding::type definition_level_encoding_;
- Encoding::type repetition_level_encoding_;
-
- bool have_def_levels_;
- bool have_rep_levels_;
- bool have_values_;
-
- // Used internally for both repetition and definition levels
- void AppendLevels(const vector<int16_t>& levels, int16_t max_level,
- Encoding::type encoding) {
- if (encoding != Encoding::RLE) {
- ParquetException::NYI("only rle encoding currently implemented");
- }
-
- // TODO: compute a more precise maximum size for the encoded levels
- vector<uint8_t> encode_buffer(levels.size() * 2);
-
- // We encode into separate memory from the output stream because the
- // RLE-encoded bytes have to be preceded in the stream by their absolute
- // size.
- LevelEncoder encoder;
- encoder.Init(encoding, max_level, static_cast<int>(levels.size()),
- encode_buffer.data(), static_cast<int>(encode_buffer.size()));
-
- encoder.Encode(static_cast<int>(levels.size()), levels.data());
-
- int32_t rle_bytes = encoder.len();
- sink_->Write(reinterpret_cast<const uint8_t*>(&rle_bytes), sizeof(int32_t));
- sink_->Write(encode_buffer.data(), rle_bytes);
- }
-};
-
-template <>
-void DataPageBuilder<BooleanType>::AppendValues(const ColumnDescriptor* d,
- const vector<bool>& values,
- Encoding::type encoding) {
- if (encoding != Encoding::PLAIN) {
- ParquetException::NYI("only plain encoding currently implemented");
- }
- PlainEncoder<BooleanType> encoder(d);
- encoder.Put(values, static_cast<int>(values.size()));
- std::shared_ptr<Buffer> buffer = encoder.FlushValues();
- sink_->Write(buffer->data(), buffer->size());
-
- num_values_ = std::max(static_cast<int32_t>(values.size()), num_values_);
- encoding_ = encoding;
- have_values_ = true;
-}
-
-template <typename Type>
-static shared_ptr<DataPage> MakeDataPage(
- const ColumnDescriptor* d, const vector<typename Type::c_type>& values, int num_vals,
- Encoding::type encoding, const uint8_t* indices, int indices_size,
- const vector<int16_t>& def_levels, int16_t max_def_level,
- const vector<int16_t>& rep_levels, int16_t max_rep_level) {
- int num_values = 0;
-
- InMemoryOutputStream page_stream;
- test::DataPageBuilder<Type> page_builder(&page_stream);
-
- if (!rep_levels.empty()) {
- page_builder.AppendRepLevels(rep_levels, max_rep_level);
- }
- if (!def_levels.empty()) {
- page_builder.AppendDefLevels(def_levels, max_def_level);
- }
-
- if (encoding == Encoding::PLAIN) {
- page_builder.AppendValues(d, values, encoding);
- num_values = page_builder.num_values();
- } else { // DICTIONARY PAGES
- page_stream.Write(indices, indices_size);
- num_values = std::max(page_builder.num_values(), num_vals);
- }
-
- auto buffer = page_stream.GetBuffer();
-
- return std::make_shared<DataPage>(buffer, num_values, encoding,
- page_builder.def_level_encoding(),
- page_builder.rep_level_encoding());
-}
-
-template <typename TYPE>
-class DictionaryPageBuilder {
- public:
- typedef typename TYPE::c_type TC;
- static constexpr int TN = TYPE::type_num;
-
- // This class writes data and metadata to the passed inputs
- explicit DictionaryPageBuilder(const ColumnDescriptor* d)
- : num_dict_values_(0), have_values_(false) {
- encoder_.reset(new DictEncoder<TYPE>(d, &pool_));
- }
-
- ~DictionaryPageBuilder() { pool_.FreeAll(); }
-
- shared_ptr<Buffer> AppendValues(const vector<TC>& values) {
- int num_values = static_cast<int>(values.size());
- // Dictionary encoding
- encoder_->Put(values.data(), num_values);
- num_dict_values_ = encoder_->num_entries();
- have_values_ = true;
- return encoder_->FlushValues();
- }
-
- shared_ptr<Buffer> WriteDict() {
- std::shared_ptr<ResizableBuffer> dict_buffer =
- AllocateBuffer(::arrow::default_memory_pool(), encoder_->dict_encoded_size());
- encoder_->WriteDict(dict_buffer->mutable_data());
- return dict_buffer;
- }
-
- int32_t num_values() const { return num_dict_values_; }
-
- private:
- ChunkedAllocator pool_;
- shared_ptr<DictEncoder<TYPE>> encoder_;
- int32_t num_dict_values_;
- bool have_values_;
-};
-
-template <>
-DictionaryPageBuilder<BooleanType>::DictionaryPageBuilder(const ColumnDescriptor* d) {
- ParquetException::NYI("only plain encoding currently implemented for boolean");
-}
-
-template <>
-shared_ptr<Buffer> DictionaryPageBuilder<BooleanType>::WriteDict() {
- ParquetException::NYI("only plain encoding currently implemented for boolean");
- return nullptr;
-}
-
-template <>
-shared_ptr<Buffer> DictionaryPageBuilder<BooleanType>::AppendValues(
- const vector<TC>& values) {
- ParquetException::NYI("only plain encoding currently implemented for boolean");
- return nullptr;
-}
-
-template <typename Type>
-static shared_ptr<DictionaryPage> MakeDictPage(
- const ColumnDescriptor* d, const vector<typename Type::c_type>& values,
- const vector<int>& values_per_page, Encoding::type encoding,
- vector<shared_ptr<Buffer>>& rle_indices) {
- InMemoryOutputStream page_stream;
- test::DictionaryPageBuilder<Type> page_builder(d);
- int num_pages = static_cast<int>(values_per_page.size());
- int value_start = 0;
-
- for (int i = 0; i < num_pages; i++) {
- rle_indices.push_back(page_builder.AppendValues(
- slice(values, value_start, value_start + values_per_page[i])));
- value_start += values_per_page[i];
- }
-
- auto buffer = page_builder.WriteDict();
-
- return std::make_shared<DictionaryPage>(buffer, page_builder.num_values(),
- Encoding::PLAIN);
-}
-
-// Given def/rep levels and values create multiple dict pages
-template <typename Type>
-static void PaginateDict(const ColumnDescriptor* d,
- const vector<typename Type::c_type>& values,
- const vector<int16_t>& def_levels, int16_t max_def_level,
- const vector<int16_t>& rep_levels, int16_t max_rep_level,
- int num_levels_per_page, const vector<int>& values_per_page,
- vector<shared_ptr<Page>>& pages,
- Encoding::type encoding = Encoding::RLE_DICTIONARY) {
- int num_pages = static_cast<int>(values_per_page.size());
- vector<shared_ptr<Buffer>> rle_indices;
- shared_ptr<DictionaryPage> dict_page =
- MakeDictPage<Type>(d, values, values_per_page, encoding, rle_indices);
- pages.push_back(dict_page);
- int def_level_start = 0;
- int def_level_end = 0;
- int rep_level_start = 0;
- int rep_level_end = 0;
- for (int i = 0; i < num_pages; i++) {
- if (max_def_level > 0) {
- def_level_start = i * num_levels_per_page;
- def_level_end = (i + 1) * num_levels_per_page;
- }
- if (max_rep_level > 0) {
- rep_level_start = i * num_levels_per_page;
- rep_level_end = (i + 1) * num_levels_per_page;
- }
- shared_ptr<DataPage> data_page = MakeDataPage<Int32Type>(
- d, {}, values_per_page[i], encoding, rle_indices[i]->data(),
- static_cast<int>(rle_indices[i]->size()),
- slice(def_levels, def_level_start, def_level_end), max_def_level,
- slice(rep_levels, rep_level_start, rep_level_end), max_rep_level);
- pages.push_back(data_page);
- }
-}
-
-// Given def/rep levels and values create multiple plain pages
-template <typename Type>
-static void PaginatePlain(const ColumnDescriptor* d,
- const vector<typename Type::c_type>& values,
- const vector<int16_t>& def_levels, int16_t max_def_level,
- const vector<int16_t>& rep_levels, int16_t max_rep_level,
- int num_levels_per_page, const vector<int>& values_per_page,
- vector<shared_ptr<Page>>& pages,
- Encoding::type encoding = Encoding::PLAIN) {
- int num_pages = static_cast<int>(values_per_page.size());
- int def_level_start = 0;
- int def_level_end = 0;
- int rep_level_start = 0;
- int rep_level_end = 0;
- int value_start = 0;
- for (int i = 0; i < num_pages; i++) {
- if (max_def_level > 0) {
- def_level_start = i * num_levels_per_page;
- def_level_end = (i + 1) * num_levels_per_page;
- }
- if (max_rep_level > 0) {
- rep_level_start = i * num_levels_per_page;
- rep_level_end = (i + 1) * num_levels_per_page;
- }
- shared_ptr<DataPage> page = MakeDataPage<Type>(
- d, slice(values, value_start, value_start + values_per_page[i]),
- values_per_page[i], encoding, nullptr, 0,
- slice(def_levels, def_level_start, def_level_end), max_def_level,
- slice(rep_levels, rep_level_start, rep_level_end), max_rep_level);
- pages.push_back(page);
- value_start += values_per_page[i];
- }
-}
-
-// Generates pages from randomly generated data
-template <typename Type>
-static int MakePages(const ColumnDescriptor* d, int num_pages, int levels_per_page,
- vector<int16_t>& def_levels, vector<int16_t>& rep_levels,
- vector<typename Type::c_type>& values, vector<uint8_t>& buffer,
- vector<shared_ptr<Page>>& pages,
- Encoding::type encoding = Encoding::PLAIN) {
- int num_levels = levels_per_page * num_pages;
- int num_values = 0;
- uint32_t seed = 0;
- int16_t zero = 0;
- int16_t max_def_level = d->max_definition_level();
- int16_t max_rep_level = d->max_repetition_level();
- vector<int> values_per_page(num_pages, levels_per_page);
- // Create definition levels
- if (max_def_level > 0) {
- def_levels.resize(num_levels);
- random_numbers(num_levels, seed, zero, max_def_level, def_levels.data());
- for (int p = 0; p < num_pages; p++) {
- int num_values_per_page = 0;
- for (int i = 0; i < levels_per_page; i++) {
- if (def_levels[i + p * levels_per_page] == max_def_level) {
- num_values_per_page++;
- num_values++;
- }
- }
- values_per_page[p] = num_values_per_page;
- }
- } else {
- num_values = num_levels;
- }
- // Create repitition levels
- if (max_rep_level > 0) {
- rep_levels.resize(num_levels);
- random_numbers(num_levels, seed, zero, max_rep_level, rep_levels.data());
- }
- // Create values
- values.resize(num_values);
- if (encoding == Encoding::PLAIN) {
- InitValues<typename Type::c_type>(num_values, values, buffer);
- PaginatePlain<Type>(d, values, def_levels, max_def_level, rep_levels, max_rep_level,
- levels_per_page, values_per_page, pages);
- } else if (encoding == Encoding::RLE_DICTIONARY ||
- encoding == Encoding::PLAIN_DICTIONARY) {
- // Calls InitValues and repeats the data
- InitDictValues<typename Type::c_type>(num_values, levels_per_page, values, buffer);
- PaginateDict<Type>(d, values, def_levels, max_def_level, rep_levels, max_rep_level,
- levels_per_page, values_per_page, pages);
- }
-
- return num_values;
-}
-
-} // namespace test
-
-} // namespace parquet
-
-#endif // PARQUET_COLUMN_TEST_UTIL_H
diff --git a/src/parquet/thrift.h b/src/parquet/thrift.h
deleted file mode 100644
index ec7ac90..0000000
--- a/src/parquet/thrift.h
+++ /dev/null
@@ -1,159 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#ifndef PARQUET_THRIFT_UTIL_H
-#define PARQUET_THRIFT_UTIL_H
-
-#include <cstdint>
-// Check if thrift version < 0.11.0
-// or if FORCE_BOOST_SMART_PTR is defined. Ref: https://thrift.apache.org/lib/cpp
-#if defined(PARQUET_THRIFT_USE_BOOST) || defined(FORCE_BOOST_SMART_PTR)
-#include <boost/shared_ptr.hpp>
-#else
-#include <memory>
-#endif
-
-// TCompactProtocol requires some #defines to work right.
-#define SIGNED_RIGHT_SHIFT_IS 1
-#define ARITHMETIC_RIGHT_SHIFT 1
-#include <thrift/TApplicationException.h>
-#include <thrift/protocol/TCompactProtocol.h>
-#include <thrift/protocol/TDebugProtocol.h>
-
-#include <thrift/protocol/TBinaryProtocol.h>
-#include <thrift/transport/TBufferTransports.h>
-#include <sstream>
-
-#include "parquet/exception.h"
-#include "parquet/parquet_types.h"
-#include "parquet/util/logging.h"
-#include "parquet/util/memory.h"
-
-namespace parquet {
-
-// Check if thrift version < 0.11.0
-// or if FORCE_BOOST_SMART_PTR is defined. Ref: https://thrift.apache.org/lib/cpp
-#if defined(PARQUET_THRIFT_USE_BOOST) || defined(FORCE_BOOST_SMART_PTR)
-using ::boost::shared_ptr;
-#else
-using ::std::shared_ptr;
-#endif
-
-// ----------------------------------------------------------------------
-// Convert Thrift enums to / from parquet enums
-
-static inline Type::type FromThrift(format::Type::type type) {
- return static_cast<Type::type>(type);
-}
-
-static inline LogicalType::type FromThrift(format::ConvertedType::type type) {
- // item 0 is NONE
- return static_cast<LogicalType::type>(static_cast<int>(type) + 1);
-}
-
-static inline Repetition::type FromThrift(format::FieldRepetitionType::type type) {
- return static_cast<Repetition::type>(type);
-}
-
-static inline Encoding::type FromThrift(format::Encoding::type type) {
- return static_cast<Encoding::type>(type);
-}
-
-static inline Compression::type FromThrift(format::CompressionCodec::type type) {
- return static_cast<Compression::type>(type);
-}
-
-static inline format::Type::type ToThrift(Type::type type) {
- return static_cast<format::Type::type>(type);
-}
-
-static inline format::ConvertedType::type ToThrift(LogicalType::type type) {
- // item 0 is NONE
- DCHECK_NE(type, LogicalType::NONE);
- return static_cast<format::ConvertedType::type>(static_cast<int>(type) - 1);
-}
-
-static inline format::FieldRepetitionType::type ToThrift(Repetition::type type) {
- return static_cast<format::FieldRepetitionType::type>(type);
-}
-
-static inline format::Encoding::type ToThrift(Encoding::type type) {
- return static_cast<format::Encoding::type>(type);
-}
-
-static inline format::CompressionCodec::type ToThrift(Compression::type type) {
- return static_cast<format::CompressionCodec::type>(type);
-}
-
-// ----------------------------------------------------------------------
-// Thrift struct serialization / deserialization utilities
-
-// Deserialize a thrift message from buf/len. buf/len must at least contain
-// all the bytes needed to store the thrift message. On return, len will be
-// set to the actual length of the header.
-template <class T>
-inline void DeserializeThriftMsg(const uint8_t* buf, uint32_t* len, T* deserialized_msg) {
- // Deserialize msg bytes into c++ thrift msg using memory transport.
- shared_ptr<apache::thrift::transport::TMemoryBuffer> tmem_transport(
- new apache::thrift::transport::TMemoryBuffer(const_cast<uint8_t*>(buf), *len));
- apache::thrift::protocol::TCompactProtocolFactoryT<
- apache::thrift::transport::TMemoryBuffer>
- tproto_factory;
- shared_ptr<apache::thrift::protocol::TProtocol> tproto =
- tproto_factory.getProtocol(tmem_transport);
- try {
- deserialized_msg->read(tproto.get());
- } catch (std::exception& e) {
- std::stringstream ss;
- ss << "Couldn't deserialize thrift: " << e.what() << "\n";
- throw ParquetException(ss.str());
- }
- uint32_t bytes_left = tmem_transport->available_read();
- *len = *len - bytes_left;
-}
-
-// Serialize obj into a buffer. The result is returned as a string.
-// The arguments are the object to be serialized and
-// the expected size of the serialized object
-template <class T>
-inline int64_t SerializeThriftMsg(T* obj, uint32_t len, OutputStream* out) {
- shared_ptr<apache::thrift::transport::TMemoryBuffer> mem_buffer(
- new apache::thrift::transport::TMemoryBuffer(len));
- apache::thrift::protocol::TCompactProtocolFactoryT<
- apache::thrift::transport::TMemoryBuffer>
- tproto_factory;
- shared_ptr<apache::thrift::protocol::TProtocol> tproto =
- tproto_factory.getProtocol(mem_buffer);
- try {
- mem_buffer->resetBuffer();
- obj->write(tproto.get());
- } catch (std::exception& e) {
- std::stringstream ss;
- ss << "Couldn't serialize thrift: " << e.what() << "\n";
- throw ParquetException(ss.str());
- }
-
- uint8_t* out_buffer;
- uint32_t out_length;
- mem_buffer->getBuffer(&out_buffer, &out_length);
- out->Write(out_buffer, out_length);
- return out_length;
-}
-
-} // namespace parquet
-
-#endif // PARQUET_THRIFT_UTIL_H
diff --git a/src/parquet/types-test.cc b/src/parquet/types-test.cc
deleted file mode 100644
index 6b184e3..0000000
--- a/src/parquet/types-test.cc
+++ /dev/null
@@ -1,142 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#include <gtest/gtest.h>
-
-#include <string>
-
-#include "parquet/types.h"
-
-namespace parquet {
-
-TEST(TestTypeToString, PhysicalTypes) {
- ASSERT_STREQ("BOOLEAN", TypeToString(Type::BOOLEAN).c_str());
- ASSERT_STREQ("INT32", TypeToString(Type::INT32).c_str());
- ASSERT_STREQ("INT64", TypeToString(Type::INT64).c_str());
- ASSERT_STREQ("INT96", TypeToString(Type::INT96).c_str());
- ASSERT_STREQ("FLOAT", TypeToString(Type::FLOAT).c_str());
- ASSERT_STREQ("DOUBLE", TypeToString(Type::DOUBLE).c_str());
- ASSERT_STREQ("BYTE_ARRAY", TypeToString(Type::BYTE_ARRAY).c_str());
- ASSERT_STREQ("FIXED_LEN_BYTE_ARRAY", TypeToString(Type::FIXED_LEN_BYTE_ARRAY).c_str());
-}
-
-TEST(TestLogicalTypeToString, LogicalTypes) {
- ASSERT_STREQ("NONE", LogicalTypeToString(LogicalType::NONE).c_str());
- ASSERT_STREQ("UTF8", LogicalTypeToString(LogicalType::UTF8).c_str());
- ASSERT_STREQ("MAP_KEY_VALUE", LogicalTypeToString(LogicalType::MAP_KEY_VALUE).c_str());
- ASSERT_STREQ("LIST", LogicalTypeToString(LogicalType::LIST).c_str());
- ASSERT_STREQ("ENUM", LogicalTypeToString(LogicalType::ENUM).c_str());
- ASSERT_STREQ("DECIMAL", LogicalTypeToString(LogicalType::DECIMAL).c_str());
- ASSERT_STREQ("DATE", LogicalTypeToString(LogicalType::DATE).c_str());
- ASSERT_STREQ("TIME_MILLIS", LogicalTypeToString(LogicalType::TIME_MILLIS).c_str());
- ASSERT_STREQ("TIME_MICROS", LogicalTypeToString(LogicalType::TIME_MICROS).c_str());
- ASSERT_STREQ("TIMESTAMP_MILLIS",
- LogicalTypeToString(LogicalType::TIMESTAMP_MILLIS).c_str());
- ASSERT_STREQ("TIMESTAMP_MICROS",
- LogicalTypeToString(LogicalType::TIMESTAMP_MICROS).c_str());
- ASSERT_STREQ("UINT_8", LogicalTypeToString(LogicalType::UINT_8).c_str());
- ASSERT_STREQ("UINT_16", LogicalTypeToString(LogicalType::UINT_16).c_str());
- ASSERT_STREQ("UINT_32", LogicalTypeToString(LogicalType::UINT_32).c_str());
- ASSERT_STREQ("UINT_64", LogicalTypeToString(LogicalType::UINT_64).c_str());
- ASSERT_STREQ("INT_8", LogicalTypeToString(LogicalType::INT_8).c_str());
- ASSERT_STREQ("INT_16", LogicalTypeToString(LogicalType::INT_16).c_str());
- ASSERT_STREQ("INT_32", LogicalTypeToString(LogicalType::INT_32).c_str());
- ASSERT_STREQ("INT_64", LogicalTypeToString(LogicalType::INT_64).c_str());
- ASSERT_STREQ("JSON", LogicalTypeToString(LogicalType::JSON).c_str());
- ASSERT_STREQ("BSON", LogicalTypeToString(LogicalType::BSON).c_str());
- ASSERT_STREQ("INTERVAL", LogicalTypeToString(LogicalType::INTERVAL).c_str());
-}
-
-TEST(TypePrinter, StatisticsTypes) {
-#if !(defined(_WIN32) || defined(__CYGWIN__))
-#pragma GCC diagnostic push
-#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
-#endif
- std::string smin;
- std::string smax;
- int32_t int_min = 1024;
- int32_t int_max = 2048;
- smin = std::string(reinterpret_cast<char*>(&int_min), sizeof(int32_t));
- smax = std::string(reinterpret_cast<char*>(&int_max), sizeof(int32_t));
- ASSERT_STREQ("1024", FormatStatValue(Type::INT32, smin).c_str());
- ASSERT_STREQ("1024", FormatStatValue(Type::INT32, smin.c_str()).c_str());
- ASSERT_STREQ("2048", FormatStatValue(Type::INT32, smax).c_str());
- ASSERT_STREQ("2048", FormatStatValue(Type::INT32, smax.c_str()).c_str());
-
- int64_t int64_min = 10240000000000;
- int64_t int64_max = 20480000000000;
- smin = std::string(reinterpret_cast<char*>(&int64_min), sizeof(int64_t));
- smax = std::string(reinterpret_cast<char*>(&int64_max), sizeof(int64_t));
- ASSERT_STREQ("10240000000000", FormatStatValue(Type::INT64, smin).c_str());
- ASSERT_STREQ("10240000000000", FormatStatValue(Type::INT64, smin.c_str()).c_str());
- ASSERT_STREQ("20480000000000", FormatStatValue(Type::INT64, smax).c_str());
- ASSERT_STREQ("20480000000000", FormatStatValue(Type::INT64, smax.c_str()).c_str());
-
- float float_min = 1.024f;
- float float_max = 2.048f;
- smin = std::string(reinterpret_cast<char*>(&float_min), sizeof(float));
- smax = std::string(reinterpret_cast<char*>(&float_max), sizeof(float));
- ASSERT_STREQ("1.024", FormatStatValue(Type::FLOAT, smin).c_str());
- ASSERT_STREQ("1.024", FormatStatValue(Type::FLOAT, smin.c_str()).c_str());
- ASSERT_STREQ("2.048", FormatStatValue(Type::FLOAT, smax).c_str());
- ASSERT_STREQ("2.048", FormatStatValue(Type::FLOAT, smax.c_str()).c_str());
-
- double double_min = 1.0245;
- double double_max = 2.0489;
- smin = std::string(reinterpret_cast<char*>(&double_min), sizeof(double));
- smax = std::string(reinterpret_cast<char*>(&double_max), sizeof(double));
- ASSERT_STREQ("1.0245", FormatStatValue(Type::DOUBLE, smin).c_str());
- ASSERT_STREQ("1.0245", FormatStatValue(Type::DOUBLE, smin.c_str()).c_str());
- ASSERT_STREQ("2.0489", FormatStatValue(Type::DOUBLE, smax).c_str());
- ASSERT_STREQ("2.0489", FormatStatValue(Type::DOUBLE, smax.c_str()).c_str());
-
- Int96 Int96_min = {{1024, 2048, 4096}};
- Int96 Int96_max = {{2048, 4096, 8192}};
- smin = std::string(reinterpret_cast<char*>(&Int96_min), sizeof(Int96));
- smax = std::string(reinterpret_cast<char*>(&Int96_max), sizeof(Int96));
- ASSERT_STREQ("1024 2048 4096", FormatStatValue(Type::INT96, smin).c_str());
- ASSERT_STREQ("1024 2048 4096", FormatStatValue(Type::INT96, smin.c_str()).c_str());
- ASSERT_STREQ("2048 4096 8192", FormatStatValue(Type::INT96, smax).c_str());
- ASSERT_STREQ("2048 4096 8192", FormatStatValue(Type::INT96, smax.c_str()).c_str());
-
- smin = std::string("abcdef");
- smax = std::string("ijklmnop");
- ASSERT_STREQ("abcdef", FormatStatValue(Type::BYTE_ARRAY, smin).c_str());
- ASSERT_STREQ("abcdef", FormatStatValue(Type::BYTE_ARRAY, smin.c_str()).c_str());
- ASSERT_STREQ("ijklmnop", FormatStatValue(Type::BYTE_ARRAY, smax).c_str());
- ASSERT_STREQ("ijklmnop", FormatStatValue(Type::BYTE_ARRAY, smax.c_str()).c_str());
-
- // PARQUET-1357: FormatStatValue truncates binary statistics on zero character
- smax.push_back('\0');
- ASSERT_EQ(smax, FormatStatValue(Type::BYTE_ARRAY, smax));
- // This fails, thus the call to FormatStatValue(.., const char*) was deprecated.
- // ASSERT_EQ(smax, FormatStatValue(Type::BYTE_ARRAY, smax.c_str()));
-
- smin = std::string("abcdefgh");
- smax = std::string("ijklmnop");
- ASSERT_STREQ("abcdefgh", FormatStatValue(Type::FIXED_LEN_BYTE_ARRAY, smin).c_str());
- ASSERT_STREQ("abcdefgh",
- FormatStatValue(Type::FIXED_LEN_BYTE_ARRAY, smin.c_str()).c_str());
- ASSERT_STREQ("ijklmnop", FormatStatValue(Type::FIXED_LEN_BYTE_ARRAY, smax).c_str());
- ASSERT_STREQ("ijklmnop",
- FormatStatValue(Type::FIXED_LEN_BYTE_ARRAY, smax.c_str()).c_str());
-#if !(defined(_WIN32) || defined(__CYGWIN__))
-#pragma GCC diagnostic pop
-#endif
-}
-
-} // namespace parquet
diff --git a/src/parquet/types.cc b/src/parquet/types.cc
deleted file mode 100644
index 3120963..0000000
--- a/src/parquet/types.cc
+++ /dev/null
@@ -1,295 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#include <cstdint>
-#include <cstring>
-#include <sstream>
-#include <string>
-
-#include "parquet/types.h"
-
-namespace parquet {
-
-std::string FormatStatValue(Type::type parquet_type, const std::string& val) {
- std::stringstream result;
- switch (parquet_type) {
- case Type::BOOLEAN:
- result << reinterpret_cast<const bool*>(val.c_str())[0];
- break;
- case Type::INT32:
- result << reinterpret_cast<const int32_t*>(val.c_str())[0];
- break;
- case Type::INT64:
- result << reinterpret_cast<const int64_t*>(val.c_str())[0];
- break;
- case Type::DOUBLE:
- result << reinterpret_cast<const double*>(val.c_str())[0];
- break;
- case Type::FLOAT:
- result << reinterpret_cast<const float*>(val.c_str())[0];
- break;
- case Type::INT96: {
- auto const i32_val = reinterpret_cast<const int32_t*>(val.c_str());
- result << i32_val[0] << " " << i32_val[1] << " " << i32_val[2];
- break;
- }
- case Type::BYTE_ARRAY: {
- return val;
- }
- case Type::FIXED_LEN_BYTE_ARRAY: {
- return val;
- }
- default:
- break;
- }
- return result.str();
-}
-
-std::string FormatStatValue(Type::type parquet_type, const char* val) {
- std::stringstream result;
- switch (parquet_type) {
- case Type::BOOLEAN:
- result << reinterpret_cast<const bool*>(val)[0];
- break;
- case Type::INT32:
- result << reinterpret_cast<const int32_t*>(val)[0];
- break;
- case Type::INT64:
- result << reinterpret_cast<const int64_t*>(val)[0];
- break;
- case Type::DOUBLE:
- result << reinterpret_cast<const double*>(val)[0];
- break;
- case Type::FLOAT:
- result << reinterpret_cast<const float*>(val)[0];
- break;
- case Type::INT96: {
- auto const i32_val = reinterpret_cast<const int32_t*>(val);
- result << i32_val[0] << " " << i32_val[1] << " " << i32_val[2];
- break;
- }
- case Type::BYTE_ARRAY: {
- result << val;
- break;
- }
- case Type::FIXED_LEN_BYTE_ARRAY: {
- result << val;
- break;
- }
- default:
- break;
- }
- return result.str();
-}
-
-std::string EncodingToString(Encoding::type t) {
- switch (t) {
- case Encoding::PLAIN:
- return "PLAIN";
- case Encoding::PLAIN_DICTIONARY:
- return "PLAIN_DICTIONARY";
- case Encoding::RLE:
- return "RLE";
- case Encoding::BIT_PACKED:
- return "BIT_PACKED";
- case Encoding::DELTA_BINARY_PACKED:
- return "DELTA_BINARY_PACKED";
- case Encoding::DELTA_LENGTH_BYTE_ARRAY:
- return "DELTA_LENGTH_BYTE_ARRAY";
- case Encoding::DELTA_BYTE_ARRAY:
- return "DELTA_BYTE_ARRAY";
- case Encoding::RLE_DICTIONARY:
- return "RLE_DICTIONARY";
- default:
- return "UNKNOWN";
- }
-}
-
-std::string CompressionToString(Compression::type t) {
- switch (t) {
- case Compression::UNCOMPRESSED:
- return "UNCOMPRESSED";
- case Compression::SNAPPY:
- return "SNAPPY";
- case Compression::GZIP:
- return "GZIP";
- case Compression::LZO:
- return "LZO";
- case Compression::LZ4:
- return "LZ4";
- case Compression::ZSTD:
- return "ZSTD";
- default:
- return "UNKNOWN";
- }
-}
-
-std::string TypeToString(Type::type t) {
- switch (t) {
- case Type::BOOLEAN:
- return "BOOLEAN";
- case Type::INT32:
- return "INT32";
- case Type::INT64:
- return "INT64";
- case Type::INT96:
- return "INT96";
- case Type::FLOAT:
- return "FLOAT";
- case Type::DOUBLE:
- return "DOUBLE";
- case Type::BYTE_ARRAY:
- return "BYTE_ARRAY";
- case Type::FIXED_LEN_BYTE_ARRAY:
- return "FIXED_LEN_BYTE_ARRAY";
- default:
- return "UNKNOWN";
- }
-}
-
-std::string LogicalTypeToString(LogicalType::type t) {
- switch (t) {
- case LogicalType::NONE:
- return "NONE";
- case LogicalType::UTF8:
- return "UTF8";
- case LogicalType::MAP_KEY_VALUE:
- return "MAP_KEY_VALUE";
- case LogicalType::LIST:
- return "LIST";
- case LogicalType::ENUM:
- return "ENUM";
- case LogicalType::DECIMAL:
- return "DECIMAL";
- case LogicalType::DATE:
- return "DATE";
- case LogicalType::TIME_MILLIS:
- return "TIME_MILLIS";
- case LogicalType::TIME_MICROS:
- return "TIME_MICROS";
- case LogicalType::TIMESTAMP_MILLIS:
- return "TIMESTAMP_MILLIS";
- case LogicalType::TIMESTAMP_MICROS:
- return "TIMESTAMP_MICROS";
- case LogicalType::UINT_8:
- return "UINT_8";
- case LogicalType::UINT_16:
- return "UINT_16";
- case LogicalType::UINT_32:
- return "UINT_32";
- case LogicalType::UINT_64:
- return "UINT_64";
- case LogicalType::INT_8:
- return "INT_8";
- case LogicalType::INT_16:
- return "INT_16";
- case LogicalType::INT_32:
- return "INT_32";
- case LogicalType::INT_64:
- return "INT_64";
- case LogicalType::JSON:
- return "JSON";
- case LogicalType::BSON:
- return "BSON";
- case LogicalType::INTERVAL:
- return "INTERVAL";
- default:
- return "UNKNOWN";
- }
-}
-
-int GetTypeByteSize(Type::type parquet_type) {
- switch (parquet_type) {
- case Type::BOOLEAN:
- return type_traits<BooleanType::type_num>::value_byte_size;
- case Type::INT32:
- return type_traits<Int32Type::type_num>::value_byte_size;
- case Type::INT64:
- return type_traits<Int64Type::type_num>::value_byte_size;
- case Type::INT96:
- return type_traits<Int96Type::type_num>::value_byte_size;
- case Type::DOUBLE:
- return type_traits<DoubleType::type_num>::value_byte_size;
- case Type::FLOAT:
- return type_traits<FloatType::type_num>::value_byte_size;
- case Type::BYTE_ARRAY:
- return type_traits<ByteArrayType::type_num>::value_byte_size;
- case Type::FIXED_LEN_BYTE_ARRAY:
- return type_traits<FLBAType::type_num>::value_byte_size;
- default:
- return 0;
- }
- return 0;
-}
-
-// Return the Sort Order of the Parquet Physical Types
-SortOrder::type DefaultSortOrder(Type::type primitive) {
- switch (primitive) {
- case Type::BOOLEAN:
- case Type::INT32:
- case Type::INT64:
- case Type::FLOAT:
- case Type::DOUBLE:
- return SortOrder::SIGNED;
- case Type::BYTE_ARRAY:
- case Type::FIXED_LEN_BYTE_ARRAY:
- return SortOrder::UNSIGNED;
- case Type::INT96:
- return SortOrder::UNKNOWN;
- }
- return SortOrder::UNKNOWN;
-}
-
-// Return the SortOrder of the Parquet Types using Logical or Physical Types
-SortOrder::type GetSortOrder(LogicalType::type converted, Type::type primitive) {
- if (converted == LogicalType::NONE) return DefaultSortOrder(primitive);
- switch (converted) {
- case LogicalType::INT_8:
- case LogicalType::INT_16:
- case LogicalType::INT_32:
- case LogicalType::INT_64:
- case LogicalType::DATE:
- case LogicalType::TIME_MICROS:
- case LogicalType::TIME_MILLIS:
- case LogicalType::TIMESTAMP_MICROS:
- case LogicalType::TIMESTAMP_MILLIS:
- return SortOrder::SIGNED;
- case LogicalType::UINT_8:
- case LogicalType::UINT_16:
- case LogicalType::UINT_32:
- case LogicalType::UINT_64:
- case LogicalType::ENUM:
- case LogicalType::UTF8:
- case LogicalType::BSON:
- case LogicalType::JSON:
- return SortOrder::UNSIGNED;
- case LogicalType::DECIMAL:
- case LogicalType::LIST:
- case LogicalType::MAP:
- case LogicalType::MAP_KEY_VALUE:
- case LogicalType::INTERVAL:
- case LogicalType::NONE: // required instead of default
- case LogicalType::NA: // required instead of default
- return SortOrder::UNKNOWN;
- }
- return SortOrder::UNKNOWN;
-}
-
-ColumnOrder ColumnOrder::undefined_ = ColumnOrder(ColumnOrder::UNDEFINED);
-ColumnOrder ColumnOrder::type_defined_ = ColumnOrder(ColumnOrder::TYPE_DEFINED_ORDER);
-
-} // namespace parquet
diff --git a/src/parquet/types.h b/src/parquet/types.h
deleted file mode 100644
index 10789cb..0000000
--- a/src/parquet/types.h
+++ /dev/null
@@ -1,316 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#ifndef PARQUET_TYPES_H
-#define PARQUET_TYPES_H
-
-#include <algorithm>
-#include <cstdint>
-#include <cstring>
-#include <iterator>
-#include <sstream>
-#include <string>
-
-#include "arrow/util/macros.h"
-
-#include "parquet/util/macros.h"
-#include "parquet/util/visibility.h"
-
-namespace parquet {
-
-// ----------------------------------------------------------------------
-// Metadata enums to match Thrift metadata
-//
-// The reason we maintain our own enums is to avoid transitive dependency on
-// the compiled Thrift headers (and thus thrift/Thrift.h) for users of the
-// public API. After building parquet-cpp, you should not need to include
-// Thrift headers in your application. This means some boilerplate to convert
-// between our types and Parquet's Thrift types.
-//
-// We can also add special values like NONE to distinguish between metadata
-// values being set and not set. As an example consider ConvertedType and
-// CompressionCodec
-
-// Mirrors parquet::Type
-struct Type {
- enum type {
- BOOLEAN = 0,
- INT32 = 1,
- INT64 = 2,
- INT96 = 3,
- FLOAT = 4,
- DOUBLE = 5,
- BYTE_ARRAY = 6,
- FIXED_LEN_BYTE_ARRAY = 7
- };
-};
-
-// Mirrors parquet::ConvertedType
-struct LogicalType {
- enum type {
- NONE,
- UTF8,
- MAP,
- MAP_KEY_VALUE,
- LIST,
- ENUM,
- DECIMAL,
- DATE,
- TIME_MILLIS,
- TIME_MICROS,
- TIMESTAMP_MILLIS,
- TIMESTAMP_MICROS,
- UINT_8,
- UINT_16,
- UINT_32,
- UINT_64,
- INT_8,
- INT_16,
- INT_32,
- INT_64,
- JSON,
- BSON,
- INTERVAL,
- NA = 25
- };
-};
-
-// Mirrors parquet::FieldRepetitionType
-struct Repetition {
- enum type { REQUIRED = 0, OPTIONAL = 1, REPEATED = 2 };
-};
-
-// Data encodings. Mirrors parquet::Encoding
-struct Encoding {
- enum type {
- PLAIN = 0,
- PLAIN_DICTIONARY = 2,
- RLE = 3,
- BIT_PACKED = 4,
- DELTA_BINARY_PACKED = 5,
- DELTA_LENGTH_BYTE_ARRAY = 6,
- DELTA_BYTE_ARRAY = 7,
- RLE_DICTIONARY = 8
- };
-};
-
-// Compression, mirrors parquet::CompressionCodec
-struct Compression {
- enum type { UNCOMPRESSED, SNAPPY, GZIP, LZO, BROTLI, LZ4, ZSTD };
-};
-
-struct Encryption {
- enum type { AES_GCM_V1 = 0, AES_GCM_CTR_V1 = 1 };
-};
-
-// parquet::PageType
-struct PageType {
- enum type { DATA_PAGE, INDEX_PAGE, DICTIONARY_PAGE, DATA_PAGE_V2 };
-};
-
-// Reference:
-// parquet-mr/parquet-hadoop/src/main/java/org/apache/parquet/
-// format/converter/ParquetMetadataConverter.java
-// Sort order for page and column statistics. Types are associated with sort
-// orders (e.g., UTF8 columns should use UNSIGNED) and column stats are
-// aggregated using a sort order. As of parquet-format version 2.3.1, the
-// order used to aggregate stats is always SIGNED and is not stored in the
-// Parquet file. These stats are discarded for types that need unsigned.
-// See PARQUET-686.
-struct SortOrder {
- enum type { SIGNED, UNSIGNED, UNKNOWN };
-};
-
-class ColumnOrder {
- public:
- enum type { UNDEFINED, TYPE_DEFINED_ORDER };
- explicit ColumnOrder(ColumnOrder::type column_order) : column_order_(column_order) {}
- // Default to Type Defined Order
- ColumnOrder() : column_order_(type::TYPE_DEFINED_ORDER) {}
- ColumnOrder::type get_order() { return column_order_; }
-
- static ColumnOrder undefined_;
- static ColumnOrder type_defined_;
-
- private:
- ColumnOrder::type column_order_;
-};
-
-// ----------------------------------------------------------------------
-
-struct ByteArray {
- ByteArray() : len(0), ptr(nullptr) {}
- ByteArray(uint32_t len, const uint8_t* ptr) : len(len), ptr(ptr) {}
- uint32_t len;
- const uint8_t* ptr;
-};
-
-inline bool operator==(const ByteArray& left, const ByteArray& right) {
- return left.len == right.len && 0 == std::memcmp(left.ptr, right.ptr, left.len);
-}
-
-inline bool operator!=(const ByteArray& left, const ByteArray& right) {
- return !(left == right);
-}
-
-struct FixedLenByteArray {
- FixedLenByteArray() : ptr(nullptr) {}
- explicit FixedLenByteArray(const uint8_t* ptr) : ptr(ptr) {}
- const uint8_t* ptr;
-};
-
-using FLBA = FixedLenByteArray;
-
-MANUALLY_ALIGNED_STRUCT(1) Int96 { uint32_t value[3]; };
-STRUCT_END(Int96, 12);
-
-inline bool operator==(const Int96& left, const Int96& right) {
- return std::equal(left.value, left.value + 3, right.value);
-}
-
-inline bool operator!=(const Int96& left, const Int96& right) { return !(left == right); }
-
-static inline std::string ByteArrayToString(const ByteArray& a) {
- return std::string(reinterpret_cast<const char*>(a.ptr), a.len);
-}
-
-static inline std::string Int96ToString(const Int96& a) {
- std::ostringstream result;
- std::copy(a.value, a.value + 3, std::ostream_iterator<uint32_t>(result, " "));
- return result.str();
-}
-
-static inline std::string FixedLenByteArrayToString(const FixedLenByteArray& a, int len) {
- std::ostringstream result;
- std::copy(a.ptr, a.ptr + len, std::ostream_iterator<uint32_t>(result, " "));
- return result.str();
-}
-
-template <Type::type TYPE>
-struct type_traits {};
-
-template <>
-struct type_traits<Type::BOOLEAN> {
- using value_type = bool;
-
- static constexpr int value_byte_size = 1;
- static constexpr const char* printf_code = "d";
-};
-
-template <>
-struct type_traits<Type::INT32> {
- using value_type = int32_t;
-
- static constexpr int value_byte_size = 4;
- static constexpr const char* printf_code = "d";
-};
-
-template <>
-struct type_traits<Type::INT64> {
- using value_type = int64_t;
-
- static constexpr int value_byte_size = 8;
- static constexpr const char* printf_code = "ld";
-};
-
-template <>
-struct type_traits<Type::INT96> {
- using value_type = Int96;
-
- static constexpr int value_byte_size = 12;
- static constexpr const char* printf_code = "s";
-};
-
-template <>
-struct type_traits<Type::FLOAT> {
- using value_type = float;
-
- static constexpr int value_byte_size = 4;
- static constexpr const char* printf_code = "f";
-};
-
-template <>
-struct type_traits<Type::DOUBLE> {
- using value_type = double;
-
- static constexpr int value_byte_size = 8;
- static constexpr const char* printf_code = "lf";
-};
-
-template <>
-struct type_traits<Type::BYTE_ARRAY> {
- using value_type = ByteArray;
-
- static constexpr int value_byte_size = sizeof(ByteArray);
- static constexpr const char* printf_code = "s";
-};
-
-template <>
-struct type_traits<Type::FIXED_LEN_BYTE_ARRAY> {
- using value_type = FixedLenByteArray;
-
- static constexpr int value_byte_size = sizeof(FixedLenByteArray);
- static constexpr const char* printf_code = "s";
-};
-
-template <Type::type TYPE>
-struct DataType {
- using c_type = typename type_traits<TYPE>::value_type;
- static constexpr Type::type type_num = TYPE;
-};
-
-using BooleanType = DataType<Type::BOOLEAN>;
-using Int32Type = DataType<Type::INT32>;
-using Int64Type = DataType<Type::INT64>;
-using Int96Type = DataType<Type::INT96>;
-using FloatType = DataType<Type::FLOAT>;
-using DoubleType = DataType<Type::DOUBLE>;
-using ByteArrayType = DataType<Type::BYTE_ARRAY>;
-using FLBAType = DataType<Type::FIXED_LEN_BYTE_ARRAY>;
-
-template <typename Type>
-inline std::string format_fwf(int width) {
- std::stringstream ss;
- ss << "%-" << width << type_traits<Type::type_num>::printf_code;
- return ss.str();
-}
-
-PARQUET_EXPORT std::string CompressionToString(Compression::type t);
-
-PARQUET_EXPORT std::string EncodingToString(Encoding::type t);
-
-PARQUET_EXPORT std::string LogicalTypeToString(LogicalType::type t);
-
-PARQUET_EXPORT std::string TypeToString(Type::type t);
-
-PARQUET_EXPORT std::string FormatStatValue(Type::type parquet_type,
- const std::string& val);
-
-/// \deprecated Since 1.5.0
-PARQUET_DEPRECATED("Use std::string instead of char* as input")
-PARQUET_EXPORT std::string FormatStatValue(Type::type parquet_type, const char* val);
-
-PARQUET_EXPORT int GetTypeByteSize(Type::type t);
-
-PARQUET_EXPORT SortOrder::type DefaultSortOrder(Type::type primitive);
-
-PARQUET_EXPORT SortOrder::type GetSortOrder(LogicalType::type converted,
- Type::type primitive);
-
-} // namespace parquet
-
-#endif // PARQUET_TYPES_H
diff --git a/src/parquet/util/CMakeLists.txt b/src/parquet/util/CMakeLists.txt
deleted file mode 100644
index 7b13886..0000000
--- a/src/parquet/util/CMakeLists.txt
+++ /dev/null
@@ -1,49 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-# Headers: util
-install(FILES
- buffer-builder.h
- comparison.h
- logging.h
- macros.h
- memory.h
- stopwatch.h
- visibility.h
- DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/parquet/util")
-
-if (PARQUET_BUILD_BENCHMARKS)
- add_library(parquet_benchmark_main benchmark_main.cc)
- if (APPLE)
- target_link_libraries(parquet_benchmark_main
- gbenchmark
- )
- elseif(WIN32)
- target_link_libraries(parquet_benchmark_main
- gbenchmark
- shlwapi.lib # workaround for bug(?) in gbenchmark: unresolved external symbol __imp_SHGetValueA
- )
- else()
- target_link_libraries(parquet_benchmark_main
- gbenchmark
- pthread
- )
- endif()
-endif()
-
-ADD_PARQUET_TEST(comparison-test)
-ADD_PARQUET_TEST(memory-test)
diff --git a/src/parquet/util/benchmark_main.cc b/src/parquet/util/benchmark_main.cc
deleted file mode 100644
index c9739af..0000000
--- a/src/parquet/util/benchmark_main.cc
+++ /dev/null
@@ -1,24 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#include "benchmark/benchmark.h"
-
-int main(int argc, char** argv) {
- benchmark::Initialize(&argc, argv);
- benchmark::RunSpecifiedBenchmarks();
- return 0;
-}
diff --git a/src/parquet/util/buffer-builder.h b/src/parquet/util/buffer-builder.h
deleted file mode 100644
index 26f134e..0000000
--- a/src/parquet/util/buffer-builder.h
+++ /dev/null
@@ -1,58 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-// Initially imported from Impala on 2016-02-23
-
-#ifndef PARQUET_UTIL_BUFFER_BUILDER_H
-#define PARQUET_UTIL_BUFFER_BUILDER_H
-
-#include <stdlib.h>
-#include <cstdint>
-
-namespace parquet {
-
-/// Utility class to build an in-memory buffer.
-class BufferBuilder {
- public:
- BufferBuilder(uint8_t* dst_buffer, int dst_len)
- : buffer_(dst_buffer), capacity_(dst_len), size_(0) {}
-
- BufferBuilder(char* dst_buffer, int dst_len)
- : buffer_(reinterpret_cast<uint8_t*>(dst_buffer)), capacity_(dst_len), size_(0) {}
-
- inline void Append(const void* buffer, int len) {
- memcpy(buffer_ + size_, buffer, len);
- size_ += len;
- }
-
- template <typename T>
- inline void Append(const T& v) {
- Append(&v, sizeof(T));
- }
-
- int capacity() const { return capacity_; }
- int size() const { return size_; }
-
- private:
- uint8_t* buffer_;
- int capacity_;
- int size_;
-};
-
-} // namespace parquet
-
-#endif // PARQUET_UTIL_BUFFER_BUILDER_H
diff --git a/src/parquet/util/comparison-test.cc b/src/parquet/util/comparison-test.cc
deleted file mode 100644
index c05f934..0000000
--- a/src/parquet/util/comparison-test.cc
+++ /dev/null
@@ -1,247 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#include <gtest/gtest.h>
-
-#include <cstdint>
-#include <iostream>
-#include <vector>
-
-#include "parquet/schema.h"
-#include "parquet/types.h"
-#include "parquet/util/comparison.h"
-
-namespace parquet {
-
-namespace test {
-
-using parquet::schema::NodePtr;
-using parquet::schema::PrimitiveNode;
-
-static ByteArray ByteArrayFromString(const std::string& s) {
- auto ptr = reinterpret_cast<const uint8_t*>(s.data());
- return ByteArray(static_cast<uint32_t>(s.size()), ptr);
-}
-
-static FLBA FLBAFromString(const std::string& s) {
- auto ptr = reinterpret_cast<const uint8_t*>(s.data());
- return FLBA(ptr);
-}
-
-TEST(Comparison, signedByteArray) {
- NodePtr node =
- PrimitiveNode::Make("SignedByteArray", Repetition::REQUIRED, Type::BYTE_ARRAY);
- ColumnDescriptor descr(node, 0, 0);
-
- CompareDefaultByteArray less;
-
- std::string s1 = "12345";
- std::string s2 = "12345678";
- ByteArray s1ba = ByteArrayFromString(s1);
- ByteArray s2ba = ByteArrayFromString(s2);
- ASSERT_TRUE(less(s1ba, s2ba));
-
- // This is case where signed comparision UTF-8 (PARQUET-686) is incorrect
- // This example is to only check signed comparison and not UTF-8.
- s1 = u8"bügeln";
- s2 = u8"braten";
- s1ba = ByteArrayFromString(s1);
- s2ba = ByteArrayFromString(s2);
- ASSERT_TRUE(less(s1ba, s2ba));
-}
-
-TEST(Comparison, UnsignedByteArray) {
- NodePtr node = PrimitiveNode::Make("UnsignedByteArray", Repetition::REQUIRED,
- Type::BYTE_ARRAY, LogicalType::UTF8);
- ColumnDescriptor descr(node, 0, 0);
-
- // Check if UTF-8 is compared using unsigned correctly
- CompareUnsignedByteArray uless;
-
- std::string s1 = "arrange";
- std::string s2 = "arrangement";
- ByteArray s1ba = ByteArrayFromString(s1);
- ByteArray s2ba = ByteArrayFromString(s2);
- ASSERT_TRUE(uless(s1ba, s2ba));
-
- // Multi-byte UTF-8 characters
- s1 = u8"braten";
- s2 = u8"bügeln";
- s1ba = ByteArrayFromString(s1);
- s2ba = ByteArrayFromString(s2);
- ASSERT_TRUE(uless(s1ba, s2ba));
-
- s1 = u8"ünk123456"; // ü = 252
- s2 = u8"ănk123456"; // ă = 259
- s1ba = ByteArrayFromString(s1);
- s2ba = ByteArrayFromString(s2);
- ASSERT_TRUE(uless(s1ba, s2ba));
-}
-
-TEST(Comparison, SignedFLBA) {
- int size = 10;
- NodePtr node = PrimitiveNode::Make("SignedFLBA", Repetition::REQUIRED,
- Type::FIXED_LEN_BYTE_ARRAY, LogicalType::NONE, size);
- ColumnDescriptor descr(node, 0, 0);
-
- CompareDefaultFLBA less(descr.type_length());
-
- std::string s1 = "Anti123456";
- std::string s2 = "Bunkd123456";
- FLBA s1flba = FLBAFromString(s1);
- FLBA s2flba = FLBAFromString(s2);
- ASSERT_TRUE(less(s1flba, s2flba));
-
- s1 = "Bünk123456";
- s2 = "Bunk123456";
- s1flba = FLBAFromString(s1);
- s2flba = FLBAFromString(s2);
- ASSERT_TRUE(less(s1flba, s2flba));
-}
-
-TEST(Comparison, UnsignedFLBA) {
- int size = 10;
- NodePtr node = PrimitiveNode::Make("UnsignedFLBA", Repetition::REQUIRED,
- Type::FIXED_LEN_BYTE_ARRAY, LogicalType::NONE, size);
- ColumnDescriptor descr(node, 0, 0);
-
- CompareUnsignedFLBA uless(descr.type_length());
-
- std::string s1 = "Anti123456";
- std::string s2 = "Bunkd123456";
- FLBA s1flba = FLBAFromString(s1);
- FLBA s2flba = FLBAFromString(s2);
- ASSERT_TRUE(uless(s1flba, s2flba));
-
- s1 = "Bunk123456";
- s2 = "Bünk123456";
- s1flba = FLBAFromString(s1);
- s2flba = FLBAFromString(s2);
- ASSERT_TRUE(uless(s1flba, s2flba));
-}
-
-TEST(Comparison, SignedInt96) {
- parquet::Int96 a{{1, 41, 14}}, b{{1, 41, 42}};
- parquet::Int96 aa{{1, 41, 14}}, bb{{1, 41, 14}};
- parquet::Int96 aaa{{1, 41, static_cast<uint32_t>(-14)}}, bbb{{1, 41, 42}};
-
- NodePtr node = PrimitiveNode::Make("SignedInt96", Repetition::REQUIRED, Type::INT96);
- ColumnDescriptor descr(node, 0, 0);
-
- CompareDefaultInt96 less;
-
- ASSERT_TRUE(less(a, b));
- ASSERT_TRUE(!less(aa, bb) && !less(bb, aa));
- ASSERT_TRUE(less(aaa, bbb));
-}
-
-TEST(Comparison, UnsignedInt96) {
- parquet::Int96 a{{1, 41, 14}}, b{{1, static_cast<uint32_t>(-41), 42}};
- parquet::Int96 aa{{1, 41, 14}}, bb{{1, 41, static_cast<uint32_t>(-14)}};
- parquet::Int96 aaa, bbb;
-
- NodePtr node = PrimitiveNode::Make("UnsignedInt96", Repetition::REQUIRED, Type::INT96);
- ColumnDescriptor descr(node, 0, 0);
-
- CompareUnsignedInt96 uless;
-
- ASSERT_TRUE(uless(a, b));
- ASSERT_TRUE(uless(aa, bb));
-
- // INT96 Timestamp
- aaa.value[2] = 2451545; // 2000-01-01
- bbb.value[2] = 2451546; // 2000-01-02
- // 12 hours + 34 minutes + 56 seconds.
- reinterpret_cast<uint64_t*>(&aaa.value[0])[0] = 45296000000000;
- // 12 hours + 34 minutes + 50 seconds.
- reinterpret_cast<uint64_t*>(&bbb.value[0])[0] = 45290000000000;
- ASSERT_TRUE(uless(aaa, bbb));
-
- aaa.value[2] = 2451545; // 2000-01-01
- bbb.value[2] = 2451545; // 2000-01-01
- // 11 hours + 34 minutes + 56 seconds.
- reinterpret_cast<uint64_t*>(&aaa.value[0])[0] = 41696000000000;
- // 12 hours + 34 minutes + 50 seconds.
- reinterpret_cast<uint64_t*>(&bbb.value[0])[0] = 45290000000000;
- ASSERT_TRUE(uless(aaa, bbb));
-
- aaa.value[2] = 2451545; // 2000-01-01
- bbb.value[2] = 2451545; // 2000-01-01
- // 12 hours + 34 minutes + 55 seconds.
- reinterpret_cast<uint64_t*>(&aaa.value[0])[0] = 45295000000000;
- // 12 hours + 34 minutes + 56 seconds.
- reinterpret_cast<uint64_t*>(&bbb.value[0])[0] = 45296000000000;
- ASSERT_TRUE(uless(aaa, bbb));
-}
-
-TEST(Comparison, SignedInt64) {
- int64_t a = 1, b = 4;
- int64_t aa = 1, bb = 1;
- int64_t aaa = -1, bbb = 1;
-
- NodePtr node = PrimitiveNode::Make("SignedInt64", Repetition::REQUIRED, Type::INT64);
- ColumnDescriptor descr(node, 0, 0);
-
- CompareDefaultInt64 less;
-
- ASSERT_TRUE(less(a, b));
- ASSERT_TRUE(!less(aa, bb) && !less(bb, aa));
- ASSERT_TRUE(less(aaa, bbb));
-}
-
-TEST(Comparison, UnsignedInt64) {
- uint64_t a = 1, b = 4;
- uint64_t aa = 1, bb = 1;
- uint64_t aaa = 1, bbb = -1;
-
- NodePtr node = PrimitiveNode::Make("UnsignedInt64", Repetition::REQUIRED, Type::INT64);
- ColumnDescriptor descr(node, 0, 0);
-
- CompareUnsignedInt64 less;
-
- ASSERT_TRUE(less(a, b));
- ASSERT_TRUE(!less(aa, bb) && !less(bb, aa));
- ASSERT_TRUE(less(aaa, bbb));
-}
-
-TEST(Comparison, UnsignedInt32) {
- uint32_t a = 1, b = 4;
- uint32_t aa = 1, bb = 1;
- uint32_t aaa = 1, bbb = -1;
-
- NodePtr node = PrimitiveNode::Make("UnsignedInt32", Repetition::REQUIRED, Type::INT32);
- ColumnDescriptor descr(node, 0, 0);
-
- CompareUnsignedInt32 less;
-
- ASSERT_TRUE(less(a, b));
- ASSERT_TRUE(!less(aa, bb) && !less(bb, aa));
- ASSERT_TRUE(less(aaa, bbb));
-}
-
-TEST(Comparison, UnknownSortOrder) {
- NodePtr node =
- PrimitiveNode::Make("Unknown", Repetition::REQUIRED, Type::FIXED_LEN_BYTE_ARRAY,
- LogicalType::INTERVAL, 12);
- ColumnDescriptor descr(node, 0, 0);
-
- ASSERT_THROW(Comparator::Make(&descr), ParquetException);
-}
-
-} // namespace test
-
-} // namespace parquet
diff --git a/src/parquet/util/comparison.cc b/src/parquet/util/comparison.cc
deleted file mode 100644
index a0768b3..0000000
--- a/src/parquet/util/comparison.cc
+++ /dev/null
@@ -1,75 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#include <algorithm>
-
-#include "parquet/exception.h"
-#include "parquet/schema.h"
-#include "parquet/types.h"
-#include "parquet/util/comparison.h"
-
-namespace parquet {
-
-std::shared_ptr<Comparator> Comparator::Make(const ColumnDescriptor* descr) {
- if (SortOrder::SIGNED == descr->sort_order()) {
- switch (descr->physical_type()) {
- case Type::BOOLEAN:
- return std::make_shared<CompareDefaultBoolean>();
- case Type::INT32:
- return std::make_shared<CompareDefaultInt32>();
- case Type::INT64:
- return std::make_shared<CompareDefaultInt64>();
- case Type::FLOAT:
- return std::make_shared<CompareDefaultFloat>();
- case Type::DOUBLE:
- return std::make_shared<CompareDefaultDouble>();
- case Type::BYTE_ARRAY:
- return std::make_shared<CompareDefaultByteArray>();
- case Type::FIXED_LEN_BYTE_ARRAY:
- return std::make_shared<CompareDefaultFLBA>(descr->type_length());
- default:
- ParquetException::NYI("Signed Compare not implemented");
- }
- } else if (SortOrder::UNSIGNED == descr->sort_order()) {
- switch (descr->physical_type()) {
- case Type::INT32:
- return std::make_shared<CompareUnsignedInt32>();
- case Type::INT64:
- return std::make_shared<CompareUnsignedInt64>();
- case Type::BYTE_ARRAY:
- return std::make_shared<CompareUnsignedByteArray>();
- case Type::FIXED_LEN_BYTE_ARRAY:
- return std::make_shared<CompareUnsignedFLBA>(descr->type_length());
- default:
- ParquetException::NYI("Unsigned Compare not implemented");
- }
- } else {
- throw ParquetException("UNKNOWN Sort Order");
- }
- return nullptr;
-}
-
-template class PARQUET_TEMPLATE_EXPORT CompareDefault<BooleanType>;
-template class PARQUET_TEMPLATE_EXPORT CompareDefault<Int32Type>;
-template class PARQUET_TEMPLATE_EXPORT CompareDefault<Int64Type>;
-template class PARQUET_TEMPLATE_EXPORT CompareDefault<Int96Type>;
-template class PARQUET_TEMPLATE_EXPORT CompareDefault<FloatType>;
-template class PARQUET_TEMPLATE_EXPORT CompareDefault<DoubleType>;
-template class PARQUET_TEMPLATE_EXPORT CompareDefault<ByteArrayType>;
-template class PARQUET_TEMPLATE_EXPORT CompareDefault<FLBAType>;
-
-} // namespace parquet
diff --git a/src/parquet/util/comparison.h b/src/parquet/util/comparison.h
deleted file mode 100644
index 7070a0f..0000000
--- a/src/parquet/util/comparison.h
+++ /dev/null
@@ -1,166 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#ifndef PARQUET_UTIL_COMPARISON_H
-#define PARQUET_UTIL_COMPARISON_H
-
-#include <algorithm>
-
-#include "parquet/exception.h"
-#include "parquet/schema.h"
-#include "parquet/types.h"
-
-namespace parquet {
-
-class PARQUET_EXPORT Comparator {
- public:
- virtual ~Comparator() {}
- static std::shared_ptr<Comparator> Make(const ColumnDescriptor* descr);
-};
-
-// The default comparison is SIGNED
-template <typename DType>
-class PARQUET_EXPORT CompareDefault : public Comparator {
- public:
- typedef typename DType::c_type T;
- CompareDefault() {}
- virtual bool operator()(const T& a, const T& b) { return a < b; }
-};
-
-template <>
-class PARQUET_EXPORT CompareDefault<Int96Type> : public Comparator {
- public:
- CompareDefault() {}
- virtual bool operator()(const Int96& a, const Int96& b) {
- // Only the MSB bit is by Signed comparison
- // For little-endian, this is the last bit of Int96 type
- const int32_t amsb = static_cast<const int32_t>(a.value[2]);
- const int32_t bmsb = static_cast<const int32_t>(b.value[2]);
- if (amsb != bmsb) {
- return (amsb < bmsb);
- } else if (a.value[1] != b.value[1]) {
- return (a.value[1] < b.value[1]);
- }
- return (a.value[0] < b.value[0]);
- }
-};
-
-template <>
-class PARQUET_EXPORT CompareDefault<ByteArrayType> : public Comparator {
- public:
- CompareDefault() {}
- virtual bool operator()(const ByteArray& a, const ByteArray& b) {
- const int8_t* aptr = reinterpret_cast<const int8_t*>(a.ptr);
- const int8_t* bptr = reinterpret_cast<const int8_t*>(b.ptr);
- return std::lexicographical_compare(aptr, aptr + a.len, bptr, bptr + b.len);
- }
-};
-
-template <>
-class PARQUET_EXPORT CompareDefault<FLBAType> : public Comparator {
- public:
- explicit CompareDefault(int length) : type_length_(length) {}
- virtual bool operator()(const FLBA& a, const FLBA& b) {
- const int8_t* aptr = reinterpret_cast<const int8_t*>(a.ptr);
- const int8_t* bptr = reinterpret_cast<const int8_t*>(b.ptr);
- return std::lexicographical_compare(aptr, aptr + type_length_, bptr,
- bptr + type_length_);
- }
- int32_t type_length_;
-};
-
-typedef CompareDefault<BooleanType> CompareDefaultBoolean;
-typedef CompareDefault<Int32Type> CompareDefaultInt32;
-typedef CompareDefault<Int64Type> CompareDefaultInt64;
-typedef CompareDefault<Int96Type> CompareDefaultInt96;
-typedef CompareDefault<FloatType> CompareDefaultFloat;
-typedef CompareDefault<DoubleType> CompareDefaultDouble;
-typedef CompareDefault<ByteArrayType> CompareDefaultByteArray;
-typedef CompareDefault<FLBAType> CompareDefaultFLBA;
-
-#if defined(__GNUC__) && !defined(__clang__)
-#pragma GCC diagnostic push
-#pragma GCC diagnostic ignored "-Wattributes"
-#endif
-
-PARQUET_EXTERN_TEMPLATE CompareDefault<BooleanType>;
-PARQUET_EXTERN_TEMPLATE CompareDefault<Int32Type>;
-PARQUET_EXTERN_TEMPLATE CompareDefault<Int64Type>;
-PARQUET_EXTERN_TEMPLATE CompareDefault<Int96Type>;
-PARQUET_EXTERN_TEMPLATE CompareDefault<FloatType>;
-PARQUET_EXTERN_TEMPLATE CompareDefault<DoubleType>;
-PARQUET_EXTERN_TEMPLATE CompareDefault<ByteArrayType>;
-PARQUET_EXTERN_TEMPLATE CompareDefault<FLBAType>;
-
-#if defined(__GNUC__) && !defined(__clang__)
-#pragma GCC diagnostic pop
-#endif
-
-// Define Unsigned Comparators
-class PARQUET_EXPORT CompareUnsignedInt32 : public CompareDefaultInt32 {
- public:
- bool operator()(const int32_t& a, const int32_t& b) override {
- const uint32_t ua = a;
- const uint32_t ub = b;
- return (ua < ub);
- }
-};
-
-class PARQUET_EXPORT CompareUnsignedInt64 : public CompareDefaultInt64 {
- public:
- bool operator()(const int64_t& a, const int64_t& b) override {
- const uint64_t ua = a;
- const uint64_t ub = b;
- return (ua < ub);
- }
-};
-
-class PARQUET_EXPORT CompareUnsignedInt96 : public CompareDefaultInt96 {
- public:
- bool operator()(const Int96& a, const Int96& b) override {
- if (a.value[2] != b.value[2]) {
- return (a.value[2] < b.value[2]);
- } else if (a.value[1] != b.value[1]) {
- return (a.value[1] < b.value[1]);
- }
- return (a.value[0] < b.value[0]);
- }
-};
-
-class PARQUET_EXPORT CompareUnsignedByteArray : public CompareDefaultByteArray {
- public:
- bool operator()(const ByteArray& a, const ByteArray& b) override {
- const uint8_t* aptr = reinterpret_cast<const uint8_t*>(a.ptr);
- const uint8_t* bptr = reinterpret_cast<const uint8_t*>(b.ptr);
- return std::lexicographical_compare(aptr, aptr + a.len, bptr, bptr + b.len);
- }
-};
-
-class PARQUET_EXPORT CompareUnsignedFLBA : public CompareDefaultFLBA {
- public:
- explicit CompareUnsignedFLBA(int length) : CompareDefaultFLBA(length) {}
- bool operator()(const FLBA& a, const FLBA& b) override {
- const uint8_t* aptr = reinterpret_cast<const uint8_t*>(a.ptr);
- const uint8_t* bptr = reinterpret_cast<const uint8_t*>(b.ptr);
- return std::lexicographical_compare(aptr, aptr + type_length_, bptr,
- bptr + type_length_);
- }
-};
-
-} // namespace parquet
-
-#endif // PARQUET_UTIL_COMPARISON_H
diff --git a/src/parquet/util/crypto.cc b/src/parquet/util/crypto.cc
deleted file mode 100644
index 59383d1..0000000
--- a/src/parquet/util/crypto.cc
+++ /dev/null
@@ -1,369 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#include "parquet/util/crypto.h"
-#include <openssl/aes.h>
-#include <openssl/evp.h>
-#include <openssl/rand.h>
-#include <algorithm>
-#include <iostream>
-#include <memory>
-#include <sstream>
-#include <string>
-#include "parquet/exception.h"
-
-using parquet::ParquetException;
-
-namespace parquet_encryption {
-
-constexpr int aesGcm = 0;
-constexpr int aesCtr = 1;
-constexpr int encryptType = 0;
-constexpr int decryptType = 1;
-constexpr int gcmTagLen = 16;
-constexpr int gcmIvLen = 12;
-constexpr int ctrIvLen = 16;
-constexpr int rndMaxBytes = 32;
-
-#define ENCRYPT_INIT(CTX, ALG) \
- if (1 != EVP_EncryptInit_ex(CTX, ALG, nullptr, nullptr, nullptr)) { \
- throw ParquetException("Couldn't init ALG encryption"); \
- }
-
-#define DECRYPT_INIT(CTX, ALG) \
- if (1 != EVP_DecryptInit_ex(CTX, ALG, nullptr, nullptr, nullptr)) { \
- throw ParquetException("Couldn't init ALG decryption"); \
- }
-
-class EvpCipher {
- public:
- explicit EvpCipher(int cipher, int key_len, int type) {
- ctx_ = nullptr;
-
- if (aesGcm != cipher && aesCtr != cipher) {
- std::stringstream ss;
- ss << "Wrong cipher: " << cipher;
- throw ParquetException(ss.str());
- }
-
- if (16 != key_len && 24 != key_len && 32 != key_len) {
- std::stringstream ss;
- ss << "Wrong key length: " << key_len;
- throw ParquetException(ss.str());
- }
-
- if (encryptType != type && decryptType != type) {
- std::stringstream ss;
- ss << "Wrong cipher type: " << type;
- throw ParquetException(ss.str());
- }
-
- ctx_ = EVP_CIPHER_CTX_new();
- if (nullptr == ctx_) {
- throw ParquetException("Couldn't init cipher context");
- }
-
- if (aesGcm == cipher) {
- // Init AES-GCM with specified key length
- if (16 == key_len) {
- if (encryptType == type) {
- ENCRYPT_INIT(ctx_, EVP_aes_128_gcm());
- } else {
- DECRYPT_INIT(ctx_, EVP_aes_128_gcm());
- }
- } else if (24 == key_len) {
- if (encryptType == type) {
- ENCRYPT_INIT(ctx_, EVP_aes_192_gcm());
- } else {
- DECRYPT_INIT(ctx_, EVP_aes_192_gcm());
- }
- } else if (32 == key_len) {
- if (encryptType == type) {
- ENCRYPT_INIT(ctx_, EVP_aes_256_gcm());
- } else {
- DECRYPT_INIT(ctx_, EVP_aes_256_gcm());
- }
- }
- } else {
- // Init AES-CTR with specified key length
- if (16 == key_len) {
- if (encryptType == type) {
- ENCRYPT_INIT(ctx_, EVP_aes_128_ctr());
- } else {
- DECRYPT_INIT(ctx_, EVP_aes_128_ctr());
- }
- } else if (24 == key_len) {
- if (encryptType == type) {
- ENCRYPT_INIT(ctx_, EVP_aes_192_ctr());
- } else {
- DECRYPT_INIT(ctx_, EVP_aes_192_ctr());
- }
- } else if (32 == key_len) {
- if (encryptType == type) {
- ENCRYPT_INIT(ctx_, EVP_aes_256_ctr());
- } else {
- DECRYPT_INIT(ctx_, EVP_aes_256_ctr());
- }
- }
- }
- }
-
- EVP_CIPHER_CTX* get() { return ctx_; }
-
- ~EvpCipher() {
- if (nullptr != ctx_) {
- EVP_CIPHER_CTX_free(ctx_);
- }
- }
-
- private:
- EVP_CIPHER_CTX* ctx_;
-};
-
-int gcm_encrypt(const uint8_t* plaintext, int plaintext_len, uint8_t* key, int key_len,
- uint8_t* aad, int aad_len, uint8_t* ciphertext) {
- int len;
- int ciphertext_len;
-
- uint8_t tag[gcmTagLen];
- memset(tag, 0, gcmTagLen);
- uint8_t iv[gcmIvLen];
- memset(iv, 0, gcmIvLen);
-
- // Random IV
- RAND_load_file("/dev/urandom", rndMaxBytes);
- RAND_bytes(iv, sizeof(iv));
-
- // Init cipher context
- EvpCipher cipher(aesGcm, key_len, encryptType);
-
- // Setting key and IV
- if (1 != EVP_EncryptInit_ex(cipher.get(), nullptr, nullptr, key, iv)) {
- throw ParquetException("Couldn't set key and IV");
- }
-
- // Setting additional authenticated data
- if ((nullptr != aad) &&
- (1 != EVP_EncryptUpdate(cipher.get(), nullptr, &len, aad, aad_len))) {
- throw ParquetException("Couldn't set AAD");
- }
-
- // Encryption
- if (1 != EVP_EncryptUpdate(cipher.get(), ciphertext + gcmIvLen, &len, plaintext,
- plaintext_len)) {
- throw ParquetException("Failed encryption update");
- }
-
- ciphertext_len = len;
-
- // Finalization
- if (1 != EVP_EncryptFinal_ex(cipher.get(), ciphertext + gcmIvLen + len, &len)) {
- throw ParquetException("Failed encryption finalization");
- }
-
- ciphertext_len += len;
-
- // Getting the tag
- if (1 != EVP_CIPHER_CTX_ctrl(cipher.get(), EVP_CTRL_GCM_GET_TAG, gcmTagLen, tag)) {
- throw ParquetException("Couldn't get AES-GCM tag");
- }
-
- // Copying the IV and tag to ciphertext
- std::copy(iv, iv + gcmIvLen, ciphertext);
- std::copy(tag, tag + gcmTagLen, ciphertext + gcmIvLen + ciphertext_len);
-
- return gcmIvLen + ciphertext_len + gcmTagLen;
-}
-
-int ctr_encrypt(const uint8_t* plaintext, int plaintext_len, uint8_t* key, int key_len,
- uint8_t* ciphertext) {
- int len;
- int ciphertext_len;
-
- uint8_t iv[ctrIvLen];
- memset(iv, 0, ctrIvLen);
-
- // Random IV
- RAND_load_file("/dev/urandom", rndMaxBytes);
- RAND_bytes(iv, sizeof(iv));
-
- // Init cipher context
- EvpCipher cipher(aesCtr, key_len, encryptType);
-
- // Setting key and IV
- if (1 != EVP_EncryptInit_ex(cipher.get(), nullptr, nullptr, key, iv)) {
- throw ParquetException("Couldn't set key and IV");
- }
-
- // Encryption
- if (1 != EVP_EncryptUpdate(cipher.get(), ciphertext + ctrIvLen, &len, plaintext,
- plaintext_len)) {
- throw ParquetException("Failed encryption update");
- }
-
- ciphertext_len = len;
-
- // Finalization
- if (1 != EVP_EncryptFinal_ex(cipher.get(), ciphertext + ctrIvLen + len, &len)) {
- throw ParquetException("Failed encryption finalization");
- }
-
- ciphertext_len += len;
-
- // Copying the IV ciphertext
- std::copy(iv, iv + ctrIvLen, ciphertext);
-
- return ctrIvLen + ciphertext_len;
-}
-
-int Encrypt(Encryption::type alg_id, bool metadata, const uint8_t* plaintext,
- int plaintext_len, uint8_t* key, int key_len, uint8_t* aad, int aad_len,
- uint8_t* ciphertext) {
- if (Encryption::AES_GCM_V1 != alg_id && Encryption::AES_GCM_CTR_V1 != alg_id) {
- std::stringstream ss;
- ss << "Crypto algorithm " << alg_id << " is not supported";
- throw ParquetException(ss.str());
- }
-
- if (metadata || (Encryption::AES_GCM_V1 == alg_id)) {
- return gcm_encrypt(plaintext, plaintext_len, key, key_len, aad, aad_len, ciphertext);
- }
-
- // Data (page) encryption with AES_GCM_CTR_V1
- return ctr_encrypt(plaintext, plaintext_len, key, key_len, ciphertext);
-}
-
-int Encrypt(std::shared_ptr<EncryptionProperties> encryption_props, bool metadata,
- const uint8_t* plaintext, int plaintext_len, uint8_t* ciphertext) {
- return Encrypt(encryption_props->algorithm(), metadata, plaintext, plaintext_len,
- encryption_props->key_bytes(), encryption_props->key_length(),
- encryption_props->aad_bytes(), encryption_props->aad_length(),
- ciphertext);
-}
-
-int gcm_decrypt(const uint8_t* ciphertext, int ciphertext_len, uint8_t* key, int key_len,
- uint8_t* aad, int aad_len, uint8_t* plaintext) {
- int len;
- int plaintext_len;
-
- uint8_t tag[gcmTagLen];
- memset(tag, 0, gcmTagLen);
- uint8_t iv[gcmIvLen];
- memset(iv, 0, gcmIvLen);
-
- // Extracting IV and tag
- std::copy(ciphertext, ciphertext + gcmIvLen, iv);
- std::copy(ciphertext + ciphertext_len - gcmTagLen, ciphertext + ciphertext_len, tag);
-
- // Init cipher context
- EvpCipher cipher(aesGcm, key_len, decryptType);
-
- // Setting key and IV
- if (1 != EVP_DecryptInit_ex(cipher.get(), nullptr, nullptr, key, iv)) {
- throw ParquetException("Couldn't set key and IV");
- }
-
- // Setting additional authenticated data
- if ((nullptr != aad) &&
- (1 != EVP_DecryptUpdate(cipher.get(), nullptr, &len, aad, aad_len))) {
- throw ParquetException("Couldn't set AAD");
- }
-
- // Decryption
- if (!EVP_DecryptUpdate(cipher.get(), plaintext, &len, ciphertext + gcmIvLen,
- ciphertext_len - gcmIvLen - gcmTagLen)) {
- throw ParquetException("Failed decryption update");
- }
-
- plaintext_len = len;
-
- // Checking the tag (authentication)
- if (!EVP_CIPHER_CTX_ctrl(cipher.get(), EVP_CTRL_GCM_SET_TAG, gcmTagLen, tag)) {
- throw ParquetException("Failed authentication");
- }
-
- // Finalization
- if (1 != EVP_DecryptFinal_ex(cipher.get(), plaintext + len, &len)) {
- throw ParquetException("Failed decryption finalization");
- }
-
- plaintext_len += len;
- return plaintext_len;
-}
-
-int ctr_decrypt(const uint8_t* ciphertext, int ciphertext_len, uint8_t* key, int key_len,
- uint8_t* plaintext) {
- int len;
- int plaintext_len;
-
- uint8_t iv[ctrIvLen];
- memset(iv, 0, ctrIvLen);
-
- // Extracting IV and tag
- std::copy(ciphertext, ciphertext + ctrIvLen, iv);
-
- // Init cipher context
- EvpCipher cipher(aesCtr, key_len, decryptType);
-
- // Setting key and IV
- if (1 != EVP_DecryptInit_ex(cipher.get(), nullptr, nullptr, key, iv)) {
- throw ParquetException("Couldn't set key and IV");
- }
-
- // Decryption
- if (!EVP_DecryptUpdate(cipher.get(), plaintext, &len, ciphertext + ctrIvLen,
- ciphertext_len - ctrIvLen)) {
- throw ParquetException("Failed decryption update");
- }
-
- plaintext_len = len;
-
- // Finalization
- if (1 != EVP_DecryptFinal_ex(cipher.get(), plaintext + len, &len)) {
- throw ParquetException("Failed decryption finalization");
- }
-
- plaintext_len += len;
- return plaintext_len;
-}
-
-int Decrypt(Encryption::type alg_id, bool metadata, const uint8_t* ciphertext,
- int ciphertext_len, uint8_t* key, int key_len, uint8_t* aad, int aad_len,
- uint8_t* plaintext) {
- if (Encryption::AES_GCM_V1 != alg_id && Encryption::AES_GCM_CTR_V1 != alg_id) {
- std::stringstream ss;
- ss << "Crypto algorithm " << alg_id << " is not supported";
- throw ParquetException(ss.str());
- }
-
- if (metadata || (Encryption::AES_GCM_V1 == alg_id)) {
- return gcm_decrypt(ciphertext, ciphertext_len, key, key_len, aad, aad_len, plaintext);
- }
-
- // Data (page) decryption with AES_GCM_CTR_V1
- return ctr_decrypt(ciphertext, ciphertext_len, key, key_len, plaintext);
-}
-
-int Decrypt(std::shared_ptr<EncryptionProperties> encryption_props, bool metadata,
- const uint8_t* ciphertext, int ciphertext_len, uint8_t* plaintext) {
- return Decrypt(encryption_props->algorithm(), metadata, ciphertext, ciphertext_len,
- encryption_props->key_bytes(), encryption_props->key_length(),
- encryption_props->aad_bytes(), encryption_props->aad_length(),
- plaintext);
-}
-
-} // namespace parquet_encryption
diff --git a/src/parquet/util/crypto.h b/src/parquet/util/crypto.h
deleted file mode 100644
index d3beb10..0000000
--- a/src/parquet/util/crypto.h
+++ /dev/null
@@ -1,44 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#ifndef PARQUET_UTIL_CRYPTO_H
-#define PARQUET_UTIL_CRYPTO_H
-
-#include "parquet/properties.h"
-#include "parquet/types.h"
-
-using parquet::Encryption;
-using parquet::EncryptionProperties;
-
-namespace parquet_encryption {
-
-int Encrypt(Encryption::type alg_id, bool metadata, const uint8_t* plaintext,
- int plaintext_len, uint8_t* key, int key_len, uint8_t* aad, int aad_len,
- uint8_t* ciphertext);
-
-int Encrypt(std::shared_ptr<EncryptionProperties> encryption_props, bool metadata,
- const uint8_t* plaintext, int plaintext_len, uint8_t* ciphertext);
-
-int Decrypt(Encryption::type alg_id, bool metadata, const uint8_t* ciphertext,
- int ciphertext_len, uint8_t* key, int key_len, uint8_t* aad, int aad_len,
- uint8_t* plaintext);
-
-int Decrypt(std::shared_ptr<EncryptionProperties> encryption_props, bool metadata,
- const uint8_t* ciphertext, int ciphertext_len, uint8_t* plaintext);
-} // namespace parquet_encryption
-
-#endif // PARQUET_UTIL_CRYPTO_H
diff --git a/src/parquet/util/logging.h b/src/parquet/util/logging.h
deleted file mode 100644
index e2c7abb..0000000
--- a/src/parquet/util/logging.h
+++ /dev/null
@@ -1,23 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#ifndef PARQUET_UTIL_LOGGING_H
-#define PARQUET_UTIL_LOGGING_H
-
-#include "arrow/util/logging.h"
-
-#endif // PARQUET_UTIL_LOGGING_H
diff --git a/src/parquet/util/macros.h b/src/parquet/util/macros.h
deleted file mode 100644
index c28b2fa..0000000
--- a/src/parquet/util/macros.h
+++ /dev/null
@@ -1,86 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#ifndef PARQUET_UTIL_MACROS_H
-#define PARQUET_UTIL_MACROS_H
-
-// Useful macros from elsewhere
-
-// From Google gutil
-#ifndef DISALLOW_COPY_AND_ASSIGN
-#define DISALLOW_COPY_AND_ASSIGN(TypeName) \
- TypeName(const TypeName&) = delete; \
- void operator=(const TypeName&) = delete
-#endif
-
-#if defined(__GNUC__)
-#define PARQUET_PREDICT_FALSE(x) (__builtin_expect(x, 0))
-#define PARQUET_PREDICT_TRUE(x) (__builtin_expect(!!(x), 1))
-#define PARQUET_NORETURN __attribute__((noreturn))
-#define PARQUET_PREFETCH(addr) __builtin_prefetch(addr)
-#elif defined(_MSC_VER)
-#define PARQUET_NORETURN __declspec(noreturn)
-#define PARQUET_PREDICT_FALSE(x) x
-#define PARQUET_PREDICT_TRUE(x) x
-#define PARQUET_PREFETCH(addr)
-#else
-#define PARQUET_NORETURN
-#define PARQUET_PREDICT_FALSE(x) x
-#define PARQUET_PREDICT_TRUE(x) x
-#define PARQUET_PREFETCH(addr)
-#endif
-
-// ----------------------------------------------------------------------
-// From googletest
-
-// When you need to test the private or protected members of a class,
-// use the FRIEND_TEST macro to declare your tests as friends of the
-// class. For example:
-//
-// class MyClass {
-// private:
-// void MyMethod();
-// FRIEND_TEST(MyClassTest, MyMethod);
-// };
-//
-// class MyClassTest : public testing::Test {
-// // ...
-// };
-//
-// TEST_F(MyClassTest, MyMethod) {
-// // Can call MyClass::MyMethod() here.
-// }
-
-#define FRIEND_TEST(test_case_name, test_name) \
- friend class test_case_name##_##test_name##_Test
-
-// clang-format off
-// [[deprecated]] is only available in C++14, use this for the time being
-// This macro takes an optional deprecation message
-#if __cplusplus <= 201103L
-# ifdef __GNUC__
-# define PARQUET_DEPRECATED(...) __attribute__((deprecated(__VA_ARGS__)))
-# elif defined(_MSC_VER)
-# define PARQUET_DEPRECATED(...) __declspec(deprecated(__VA_ARGS__))
-# else
-# define PARQUET_DEPRECATED(...)
-# endif
-#else
-# define PARQUET_DEPRECATED(...) [[deprecated(__VA_ARGS__)]]
-#endif
-
-#endif // PARQUET_UTIL_MACROS_H
diff --git a/src/parquet/util/memory-test.cc b/src/parquet/util/memory-test.cc
deleted file mode 100644
index bfd685d..0000000
--- a/src/parquet/util/memory-test.cc
+++ /dev/null
@@ -1,362 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#include <cstdint>
-#include <cstdio>
-#include <memory>
-#include <string>
-#include <vector>
-
-#include <gtest/gtest.h>
-
-#include "parquet/exception.h"
-#include "parquet/util/memory.h"
-#include "parquet/util/test-common.h"
-
-using arrow::default_memory_pool;
-using arrow::MemoryPool;
-
-namespace parquet {
-
-class TestBuffer : public ::testing::Test {};
-
-// Utility class to call private functions on MemPool.
-class ChunkedAllocatorTest {
- public:
- static bool CheckIntegrity(ChunkedAllocator* pool, bool current_chunk_empty) {
- return pool->CheckIntegrity(current_chunk_empty);
- }
-
- static const int INITIAL_CHUNK_SIZE = ChunkedAllocator::INITIAL_CHUNK_SIZE;
- static const int MAX_CHUNK_SIZE = ChunkedAllocator::MAX_CHUNK_SIZE;
-};
-
-const int ChunkedAllocatorTest::INITIAL_CHUNK_SIZE;
-const int ChunkedAllocatorTest::MAX_CHUNK_SIZE;
-
-TEST(ChunkedAllocatorTest, Basic) {
- ChunkedAllocator p;
- ChunkedAllocator p2;
- ChunkedAllocator p3;
-
- for (int iter = 0; iter < 2; ++iter) {
- // allocate a total of 24K in 32-byte pieces (for which we only request 25 bytes)
- for (int i = 0; i < 768; ++i) {
- // pads to 32 bytes
- p.Allocate(25);
- }
- // we handed back 24K
- EXPECT_EQ(24 * 1024, p.total_allocated_bytes());
- // .. and allocated 28K of chunks (4, 8, 16)
- EXPECT_EQ(28 * 1024, p.GetTotalChunkSizes());
-
- // we're passing on the first two chunks, containing 12K of data; we're left with
- // one chunk of 16K containing 12K of data
- p2.AcquireData(&p, true);
- EXPECT_EQ(12 * 1024, p.total_allocated_bytes());
- EXPECT_EQ(16 * 1024, p.GetTotalChunkSizes());
-
- // we allocate 8K, for which there isn't enough room in the current chunk,
- // so another one is allocated (32K)
- p.Allocate(8 * 1024);
- EXPECT_EQ((16 + 32) * 1024, p.GetTotalChunkSizes());
-
- // we allocate 65K, which doesn't fit into the current chunk or the default
- // size of the next allocated chunk (64K)
- p.Allocate(65 * 1024);
- EXPECT_EQ((12 + 8 + 65) * 1024, p.total_allocated_bytes());
- if (iter == 0) {
- EXPECT_EQ((12 + 8 + 65) * 1024, p.peak_allocated_bytes());
- } else {
- EXPECT_EQ((1 + 120 + 33) * 1024, p.peak_allocated_bytes());
- }
- EXPECT_EQ((16 + 32 + 65) * 1024, p.GetTotalChunkSizes());
-
- // Clear() resets allocated data, but doesn't remove any chunks
- p.Clear();
- EXPECT_EQ(0, p.total_allocated_bytes());
- if (iter == 0) {
- EXPECT_EQ((12 + 8 + 65) * 1024, p.peak_allocated_bytes());
- } else {
- EXPECT_EQ((1 + 120 + 33) * 1024, p.peak_allocated_bytes());
- }
- EXPECT_EQ((16 + 32 + 65) * 1024, p.GetTotalChunkSizes());
-
- // next allocation reuses existing chunks
- p.Allocate(1024);
- EXPECT_EQ(1024, p.total_allocated_bytes());
- if (iter == 0) {
- EXPECT_EQ((12 + 8 + 65) * 1024, p.peak_allocated_bytes());
- } else {
- EXPECT_EQ((1 + 120 + 33) * 1024, p.peak_allocated_bytes());
- }
- EXPECT_EQ((16 + 32 + 65) * 1024, p.GetTotalChunkSizes());
-
- // ... unless it doesn't fit into any available chunk
- p.Allocate(120 * 1024);
- EXPECT_EQ((1 + 120) * 1024, p.total_allocated_bytes());
- if (iter == 0) {
- EXPECT_EQ((1 + 120) * 1024, p.peak_allocated_bytes());
- } else {
- EXPECT_EQ((1 + 120 + 33) * 1024, p.peak_allocated_bytes());
- }
- EXPECT_EQ((130 + 16 + 32 + 65) * 1024, p.GetTotalChunkSizes());
-
- // ... Try another chunk that fits into an existing chunk
- p.Allocate(33 * 1024);
- EXPECT_EQ((1 + 120 + 33) * 1024, p.total_allocated_bytes());
- EXPECT_EQ((130 + 16 + 32 + 65) * 1024, p.GetTotalChunkSizes());
-
- // we're releasing 3 chunks, which get added to p2
- p2.AcquireData(&p, false);
- EXPECT_EQ(0, p.total_allocated_bytes());
- EXPECT_EQ((1 + 120 + 33) * 1024, p.peak_allocated_bytes());
- EXPECT_EQ(0, p.GetTotalChunkSizes());
-
- p3.AcquireData(&p2, true); // we're keeping the 65k chunk
- EXPECT_EQ(33 * 1024, p2.total_allocated_bytes());
- EXPECT_EQ(65 * 1024, p2.GetTotalChunkSizes());
-
- p.FreeAll();
- p2.FreeAll();
- p3.FreeAll();
- }
-}
-
-// Test that we can keep an allocated chunk and a free chunk.
-// This case verifies that when chunks are acquired by another memory pool the
-// remaining chunks are consistent if there were more than one used chunk and some
-// free chunks.
-TEST(ChunkedAllocatorTest, Keep) {
- ChunkedAllocator p;
- p.Allocate(4 * 1024);
- p.Allocate(8 * 1024);
- p.Allocate(16 * 1024);
- EXPECT_EQ((4 + 8 + 16) * 1024, p.total_allocated_bytes());
- EXPECT_EQ((4 + 8 + 16) * 1024, p.GetTotalChunkSizes());
- p.Clear();
- EXPECT_EQ(0, p.total_allocated_bytes());
- EXPECT_EQ((4 + 8 + 16) * 1024, p.GetTotalChunkSizes());
- p.Allocate(1 * 1024);
- p.Allocate(4 * 1024);
- EXPECT_EQ((1 + 4) * 1024, p.total_allocated_bytes());
- EXPECT_EQ((4 + 8 + 16) * 1024, p.GetTotalChunkSizes());
-
- ChunkedAllocator p2;
- p2.AcquireData(&p, true);
- EXPECT_EQ(4 * 1024, p.total_allocated_bytes());
- EXPECT_EQ((8 + 16) * 1024, p.GetTotalChunkSizes());
- EXPECT_EQ(1 * 1024, p2.total_allocated_bytes());
- EXPECT_EQ(4 * 1024, p2.GetTotalChunkSizes());
-
- p.FreeAll();
- p2.FreeAll();
-}
-
-// Tests that we can return partial allocations.
-TEST(ChunkedAllocatorTest, ReturnPartial) {
- ChunkedAllocator p;
- uint8_t* ptr = p.Allocate(1024);
- EXPECT_EQ(1024, p.total_allocated_bytes());
- memset(ptr, 0, 1024);
- p.ReturnPartialAllocation(1024);
-
- uint8_t* ptr2 = p.Allocate(1024);
- EXPECT_EQ(1024, p.total_allocated_bytes());
- EXPECT_TRUE(ptr == ptr2);
- p.ReturnPartialAllocation(1016);
-
- ptr2 = p.Allocate(1016);
- EXPECT_EQ(1024, p.total_allocated_bytes());
- EXPECT_TRUE(ptr2 == ptr + 8);
- p.ReturnPartialAllocation(512);
- memset(ptr2, 1, 1016 - 512);
-
- uint8_t* ptr3 = p.Allocate(512);
- EXPECT_EQ(1024, p.total_allocated_bytes());
- EXPECT_TRUE(ptr3 == ptr + 512);
- memset(ptr3, 2, 512);
-
- for (int i = 0; i < 8; ++i) {
- EXPECT_EQ(0, ptr[i]);
- }
- for (int i = 8; i < 512; ++i) {
- EXPECT_EQ(1, ptr[i]);
- }
- for (int i = 512; i < 1024; ++i) {
- EXPECT_EQ(2, ptr[i]);
- }
-
- p.FreeAll();
-}
-
-// Test that the ChunkedAllocator overhead is bounded when we make allocations of
-// INITIAL_CHUNK_SIZE.
-TEST(ChunkedAllocatorTest, MemoryOverhead) {
- ChunkedAllocator p;
- const int alloc_size = ChunkedAllocatorTest::INITIAL_CHUNK_SIZE;
- const int num_allocs = 1000;
- int64_t total_allocated = 0;
-
- for (int i = 0; i < num_allocs; ++i) {
- uint8_t* mem = p.Allocate(alloc_size);
- ASSERT_TRUE(mem != nullptr);
- total_allocated += alloc_size;
-
- int64_t wasted_memory = p.GetTotalChunkSizes() - total_allocated;
- // The initial chunk fits evenly into MAX_CHUNK_SIZE, so should have at most
- // one empty chunk at the end.
- EXPECT_LE(wasted_memory, ChunkedAllocatorTest::MAX_CHUNK_SIZE);
- // The chunk doubling algorithm should not allocate chunks larger than the total
- // amount of memory already allocated.
- EXPECT_LE(wasted_memory, total_allocated);
- }
-
- p.FreeAll();
-}
-
-// Test that the ChunkedAllocator overhead is bounded when we make alternating
-// large and small allocations.
-TEST(ChunkedAllocatorTest, FragmentationOverhead) {
- ChunkedAllocator p;
- const int num_allocs = 100;
- int64_t total_allocated = 0;
-
- for (int i = 0; i < num_allocs; ++i) {
- int alloc_size = i % 2 == 0 ? 1 : ChunkedAllocatorTest::MAX_CHUNK_SIZE;
- uint8_t* mem = p.Allocate(alloc_size);
- ASSERT_TRUE(mem != nullptr);
- total_allocated += alloc_size;
-
- int64_t wasted_memory = p.GetTotalChunkSizes() - total_allocated;
- // Fragmentation should not waste more than half of each completed chunk.
- EXPECT_LE(wasted_memory, total_allocated + ChunkedAllocatorTest::MAX_CHUNK_SIZE);
- }
-
- p.FreeAll();
-}
-
-TEST(TestBufferedInputStream, Basics) {
- int64_t source_size = 256;
- int64_t stream_offset = 10;
- int64_t stream_size = source_size - stream_offset;
- int64_t chunk_size = 50;
- std::shared_ptr<ResizableBuffer> buf =
- AllocateBuffer(default_memory_pool(), source_size);
- ASSERT_EQ(source_size, buf->size());
- for (int i = 0; i < source_size; i++) {
- buf->mutable_data()[i] = static_cast<uint8_t>(i);
- }
-
- auto wrapper =
- std::make_shared<ArrowInputFile>(std::make_shared<::arrow::io::BufferReader>(buf));
-
- std::unique_ptr<BufferedInputStream> stream(new BufferedInputStream(
- default_memory_pool(), chunk_size, wrapper.get(), stream_offset, stream_size));
-
- const uint8_t* output;
- int64_t bytes_read;
-
- // source is at offset 10
- output = stream->Peek(10, &bytes_read);
- ASSERT_EQ(10, bytes_read);
- for (int i = 0; i < 10; i++) {
- ASSERT_EQ(10 + i, output[i]) << i;
- }
- output = stream->Read(10, &bytes_read);
- ASSERT_EQ(10, bytes_read);
- for (int i = 0; i < 10; i++) {
- ASSERT_EQ(10 + i, output[i]) << i;
- }
- output = stream->Read(10, &bytes_read);
- ASSERT_EQ(10, bytes_read);
- for (int i = 0; i < 10; i++) {
- ASSERT_EQ(20 + i, output[i]) << i;
- }
- stream->Advance(5);
- stream->Advance(5);
- // source is at offset 40
- // read across buffer boundary. buffer size is 50
- output = stream->Read(20, &bytes_read);
- ASSERT_EQ(20, bytes_read);
- for (int i = 0; i < 20; i++) {
- ASSERT_EQ(40 + i, output[i]) << i;
- }
- // read more than original chunk_size
- output = stream->Read(60, &bytes_read);
- ASSERT_EQ(60, bytes_read);
- for (int i = 0; i < 60; i++) {
- ASSERT_EQ(60 + i, output[i]) << i;
- }
-
- stream->Advance(120);
- // source is at offset 240
- // read outside of source boundary. source size is 256
- output = stream->Read(30, &bytes_read);
- ASSERT_EQ(16, bytes_read);
- for (int i = 0; i < 16; i++) {
- ASSERT_EQ(240 + i, output[i]) << i;
- }
-}
-
-TEST(TestArrowInputFile, ReadAt) {
- std::string data = "this is the data";
- auto data_buffer = reinterpret_cast<const uint8_t*>(data.c_str());
-
- auto file = std::make_shared<::arrow::io::BufferReader>(data_buffer, data.size());
- auto source = std::make_shared<ArrowInputFile>(file);
-
- ASSERT_EQ(0, source->Tell());
-
- uint8_t buffer[50];
-
- ASSERT_NO_THROW(source->ReadAt(0, 4, buffer));
- ASSERT_EQ(0, std::memcmp(buffer, "this", 4));
-
- // Note: it's undefined (and possibly platform-dependent) whether ArrowInputFile
- // updates the file position after ReadAt().
-}
-
-TEST(TestArrowInputFile, Read) {
- std::string data = "this is the data";
- auto data_buffer = reinterpret_cast<const uint8_t*>(data.c_str());
-
- auto file = std::make_shared<::arrow::io::BufferReader>(data_buffer, data.size());
- auto source = std::make_shared<ArrowInputFile>(file);
-
- ASSERT_EQ(0, source->Tell());
-
- std::shared_ptr<Buffer> pq_buffer, expected_buffer;
-
- ASSERT_NO_THROW(pq_buffer = source->Read(4));
- expected_buffer = std::make_shared<Buffer>(data_buffer, 4);
- ASSERT_TRUE(expected_buffer->Equals(*pq_buffer.get()));
-
- ASSERT_NO_THROW(pq_buffer = source->Read(7));
- expected_buffer = std::make_shared<Buffer>(data_buffer + 4, 7);
- ASSERT_TRUE(expected_buffer->Equals(*pq_buffer.get()));
-
- ASSERT_EQ(11, source->Tell());
-
- ASSERT_NO_THROW(pq_buffer = source->Read(8));
- expected_buffer = std::make_shared<Buffer>(data_buffer + 11, 5);
- ASSERT_TRUE(expected_buffer->Equals(*pq_buffer.get()));
-
- ASSERT_EQ(16, source->Tell());
-}
-
-} // namespace parquet
diff --git a/src/parquet/util/memory.cc b/src/parquet/util/memory.cc
deleted file mode 100644
index d9caf6e..0000000
--- a/src/parquet/util/memory.cc
+++ /dev/null
@@ -1,502 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#include "parquet/util/memory.h"
-
-#include <algorithm>
-#include <cstdint>
-#include <cstdio>
-#include <string>
-#include <utility>
-
-#include "arrow/status.h"
-#include "arrow/util/bit-util.h"
-
-#include "parquet/exception.h"
-#include "parquet/types.h"
-#include "parquet/util/logging.h"
-
-using arrow::MemoryPool;
-
-namespace parquet {
-
-template <class T>
-Vector<T>::Vector(int64_t size, MemoryPool* pool)
- : buffer_(AllocateBuffer(pool, size * sizeof(T))), size_(size), capacity_(size) {
- if (size > 0) {
- data_ = reinterpret_cast<T*>(buffer_->mutable_data());
- } else {
- data_ = nullptr;
- }
-}
-
-template <class T>
-void Vector<T>::Reserve(int64_t new_capacity) {
- if (new_capacity > capacity_) {
- PARQUET_THROW_NOT_OK(buffer_->Resize(new_capacity * sizeof(T)));
- data_ = reinterpret_cast<T*>(buffer_->mutable_data());
- capacity_ = new_capacity;
- }
-}
-
-template <class T>
-void Vector<T>::Resize(int64_t new_size) {
- Reserve(new_size);
- size_ = new_size;
-}
-
-template <class T>
-void Vector<T>::Assign(int64_t size, const T val) {
- Resize(size);
- for (int64_t i = 0; i < size_; i++) {
- data_[i] = val;
- }
-}
-
-template <class T>
-void Vector<T>::Swap(Vector<T>& v) {
- buffer_.swap(v.buffer_);
- std::swap(size_, v.size_);
- std::swap(capacity_, v.capacity_);
- std::swap(data_, v.data_);
-}
-
-template class Vector<int32_t>;
-template class Vector<int64_t>;
-template class Vector<bool>;
-template class Vector<float>;
-template class Vector<double>;
-template class Vector<Int96>;
-template class Vector<ByteArray>;
-template class Vector<FixedLenByteArray>;
-
-const int ChunkedAllocator::INITIAL_CHUNK_SIZE;
-const int ChunkedAllocator::MAX_CHUNK_SIZE;
-
-ChunkedAllocator::ChunkedAllocator(MemoryPool* pool)
- : current_chunk_idx_(-1),
- next_chunk_size_(INITIAL_CHUNK_SIZE),
- total_allocated_bytes_(0),
- peak_allocated_bytes_(0),
- total_reserved_bytes_(0),
- pool_(pool) {}
-
-ChunkedAllocator::ChunkInfo::ChunkInfo(int64_t size, uint8_t* buf)
- : data(buf), size(size), allocated_bytes(0) {}
-
-ChunkedAllocator::~ChunkedAllocator() {
- int64_t total_bytes_released = 0;
- for (size_t i = 0; i < chunks_.size(); ++i) {
- total_bytes_released += chunks_[i].size;
- pool_->Free(chunks_[i].data, chunks_[i].size);
- }
-
- DCHECK(chunks_.empty()) << "Must call FreeAll() or AcquireData() for this pool";
-}
-
-void ChunkedAllocator::ReturnPartialAllocation(int byte_size) {
- DCHECK_GE(byte_size, 0);
- DCHECK(current_chunk_idx_ != -1);
- ChunkInfo& info = chunks_[current_chunk_idx_];
- DCHECK_GE(info.allocated_bytes, byte_size);
- info.allocated_bytes -= byte_size;
- total_allocated_bytes_ -= byte_size;
-}
-
-template <bool CHECK_LIMIT_FIRST>
-uint8_t* ChunkedAllocator::Allocate(int size) {
- if (size == 0) {
- return nullptr;
- }
-
- int64_t num_bytes = ::arrow::BitUtil::RoundUp(size, 8);
- if (current_chunk_idx_ == -1 ||
- num_bytes + chunks_[current_chunk_idx_].allocated_bytes >
- chunks_[current_chunk_idx_].size) {
- // If we couldn't allocate a new chunk, return nullptr.
- if (ARROW_PREDICT_FALSE(!FindChunk(num_bytes))) {
- return nullptr;
- }
- }
- ChunkInfo& info = chunks_[current_chunk_idx_];
- uint8_t* result = info.data + info.allocated_bytes;
- DCHECK_LE(info.allocated_bytes + num_bytes, info.size);
- info.allocated_bytes += num_bytes;
- total_allocated_bytes_ += num_bytes;
- DCHECK_LE(current_chunk_idx_, static_cast<int>(chunks_.size()) - 1);
- peak_allocated_bytes_ = std::max(total_allocated_bytes_, peak_allocated_bytes_);
- return result;
-}
-
-uint8_t* ChunkedAllocator::Allocate(int size) { return Allocate<false>(size); }
-
-void ChunkedAllocator::Clear() {
- current_chunk_idx_ = -1;
- for (auto chunk = chunks_.begin(); chunk != chunks_.end(); ++chunk) {
- chunk->allocated_bytes = 0;
- }
- total_allocated_bytes_ = 0;
- DCHECK(CheckIntegrity(false));
-}
-
-void ChunkedAllocator::FreeAll() {
- int64_t total_bytes_released = 0;
- for (size_t i = 0; i < chunks_.size(); ++i) {
- total_bytes_released += chunks_[i].size;
- pool_->Free(chunks_[i].data, chunks_[i].size);
- }
- chunks_.clear();
- next_chunk_size_ = INITIAL_CHUNK_SIZE;
- current_chunk_idx_ = -1;
- total_allocated_bytes_ = 0;
- total_reserved_bytes_ = 0;
-}
-
-bool ChunkedAllocator::FindChunk(int64_t min_size) {
- // Try to allocate from a free chunk. The first free chunk, if any, will be immediately
- // after the current chunk.
- int first_free_idx = current_chunk_idx_ + 1;
- // (cast size() to signed int in order to avoid everything else being cast to
- // unsigned long, in particular -1)
- while (++current_chunk_idx_ < static_cast<int>(chunks_.size())) {
- // we found a free chunk
- DCHECK_EQ(chunks_[current_chunk_idx_].allocated_bytes, 0);
-
- if (chunks_[current_chunk_idx_].size >= min_size) {
- // This chunk is big enough. Move it before the other free chunks.
- if (current_chunk_idx_ != first_free_idx) {
- std::swap(chunks_[current_chunk_idx_], chunks_[first_free_idx]);
- current_chunk_idx_ = first_free_idx;
- }
- break;
- }
- }
-
- if (current_chunk_idx_ == static_cast<int>(chunks_.size())) {
- // need to allocate new chunk.
- int64_t chunk_size;
- DCHECK_GE(next_chunk_size_, INITIAL_CHUNK_SIZE);
- DCHECK_LE(next_chunk_size_, MAX_CHUNK_SIZE);
-
- chunk_size = std::max<int64_t>(min_size, next_chunk_size_);
-
- // Allocate a new chunk. Return early if malloc fails.
- uint8_t* buf = nullptr;
- PARQUET_THROW_NOT_OK(pool_->Allocate(chunk_size, &buf));
- if (ARROW_PREDICT_FALSE(buf == nullptr)) {
- DCHECK_EQ(current_chunk_idx_, static_cast<int>(chunks_.size()));
- current_chunk_idx_ = static_cast<int>(chunks_.size()) - 1;
- return false;
- }
-
- // If there are no free chunks put it at the end, otherwise before the first free.
- if (first_free_idx == static_cast<int>(chunks_.size())) {
- chunks_.push_back(ChunkInfo(chunk_size, buf));
- } else {
- current_chunk_idx_ = first_free_idx;
- auto insert_chunk = chunks_.begin() + current_chunk_idx_;
- chunks_.insert(insert_chunk, ChunkInfo(chunk_size, buf));
- }
- total_reserved_bytes_ += chunk_size;
- // Don't increment the chunk size until the allocation succeeds: if an attempted
- // large allocation fails we don't want to increase the chunk size further.
- next_chunk_size_ =
- static_cast<int>(std::min<int64_t>(chunk_size * 2, MAX_CHUNK_SIZE));
- }
-
- DCHECK_LT(current_chunk_idx_, static_cast<int>(chunks_.size()));
- DCHECK(CheckIntegrity(true));
- return true;
-}
-
-void ChunkedAllocator::AcquireData(ChunkedAllocator* src, bool keep_current) {
- DCHECK(src->CheckIntegrity(false));
- int num_acquired_chunks;
- if (keep_current) {
- num_acquired_chunks = src->current_chunk_idx_;
- } else if (src->GetFreeOffset() == 0) {
- // nothing in the last chunk
- num_acquired_chunks = src->current_chunk_idx_;
- } else {
- num_acquired_chunks = src->current_chunk_idx_ + 1;
- }
-
- if (num_acquired_chunks <= 0) {
- if (!keep_current) src->FreeAll();
- return;
- }
-
- auto end_chunk = src->chunks_.begin() + num_acquired_chunks;
- int64_t total_transfered_bytes = 0;
- for (auto i = src->chunks_.begin(); i != end_chunk; ++i) {
- total_transfered_bytes += i->size;
- }
- src->total_reserved_bytes_ -= total_transfered_bytes;
- total_reserved_bytes_ += total_transfered_bytes;
-
- // insert new chunks after current_chunk_idx_
- auto insert_chunk = chunks_.begin() + (current_chunk_idx_ + 1);
- chunks_.insert(insert_chunk, src->chunks_.begin(), end_chunk);
- src->chunks_.erase(src->chunks_.begin(), end_chunk);
- current_chunk_idx_ += num_acquired_chunks;
-
- if (keep_current) {
- src->current_chunk_idx_ = 0;
- DCHECK(src->chunks_.size() == 1 || src->chunks_[1].allocated_bytes == 0);
- total_allocated_bytes_ += src->total_allocated_bytes_ - src->GetFreeOffset();
- src->total_allocated_bytes_ = src->GetFreeOffset();
- } else {
- src->current_chunk_idx_ = -1;
- total_allocated_bytes_ += src->total_allocated_bytes_;
- src->total_allocated_bytes_ = 0;
- }
- peak_allocated_bytes_ = std::max(total_allocated_bytes_, peak_allocated_bytes_);
-
- if (!keep_current) src->FreeAll();
- DCHECK(CheckIntegrity(false));
-}
-
-std::string ChunkedAllocator::DebugString() {
- std::stringstream out;
- char str[16];
- out << "ChunkedAllocator(#chunks=" << chunks_.size() << " [";
- for (size_t i = 0; i < chunks_.size(); ++i) {
- sprintf(str, "0x%zx=", reinterpret_cast<size_t>(chunks_[i].data)); // NOLINT
- out << (i > 0 ? " " : "") << str << chunks_[i].size << "/"
- << chunks_[i].allocated_bytes;
- }
- out << "] current_chunk=" << current_chunk_idx_
- << " total_sizes=" << GetTotalChunkSizes()
- << " total_alloc=" << total_allocated_bytes_ << ")";
- return out.str();
-}
-
-int64_t ChunkedAllocator::GetTotalChunkSizes() const {
- int64_t result = 0;
- for (size_t i = 0; i < chunks_.size(); ++i) {
- result += chunks_[i].size;
- }
- return result;
-}
-
-bool ChunkedAllocator::CheckIntegrity(bool current_chunk_empty) {
- // check that current_chunk_idx_ points to the last chunk with allocated data
- DCHECK_LT(current_chunk_idx_, static_cast<int>(chunks_.size()));
- int64_t total_allocated = 0;
- for (int i = 0; i < static_cast<int>(chunks_.size()); ++i) {
- DCHECK_GT(chunks_[i].size, 0);
- if (i < current_chunk_idx_) {
- DCHECK_GT(chunks_[i].allocated_bytes, 0);
- } else if (i == current_chunk_idx_) {
- if (current_chunk_empty) {
- DCHECK_EQ(chunks_[i].allocated_bytes, 0);
- } else {
- DCHECK_GT(chunks_[i].allocated_bytes, 0);
- }
- } else {
- DCHECK_EQ(chunks_[i].allocated_bytes, 0);
- }
- total_allocated += chunks_[i].allocated_bytes;
- }
- DCHECK_EQ(total_allocated, total_allocated_bytes_);
- return true;
-}
-
-// ----------------------------------------------------------------------
-// Arrow IO wrappers
-
-void ArrowFileMethods::Close() {
- // Closing the file is the responsibility of the owner of the handle
- return;
-}
-
-// Return the current position in the output stream relative to the start
-int64_t ArrowFileMethods::Tell() {
- int64_t position = 0;
- PARQUET_THROW_NOT_OK(file_interface()->Tell(&position));
- return position;
-}
-
-ArrowInputFile::ArrowInputFile(
- const std::shared_ptr<::arrow::io::ReadableFileInterface>& file)
- : file_(file) {}
-
-::arrow::io::FileInterface* ArrowInputFile::file_interface() { return file_.get(); }
-
-int64_t ArrowInputFile::Size() const {
- int64_t size;
- PARQUET_THROW_NOT_OK(file_->GetSize(&size));
- return size;
-}
-
-// Returns bytes read
-int64_t ArrowInputFile::Read(int64_t nbytes, uint8_t* out) {
- int64_t bytes_read = 0;
- PARQUET_THROW_NOT_OK(file_->Read(nbytes, &bytes_read, out));
- return bytes_read;
-}
-
-std::shared_ptr<Buffer> ArrowInputFile::Read(int64_t nbytes) {
- std::shared_ptr<Buffer> out;
- PARQUET_THROW_NOT_OK(file_->Read(nbytes, &out));
- return out;
-}
-
-std::shared_ptr<Buffer> ArrowInputFile::ReadAt(int64_t position, int64_t nbytes) {
- std::shared_ptr<Buffer> out;
- PARQUET_THROW_NOT_OK(file_->ReadAt(position, nbytes, &out));
- return out;
-}
-
-int64_t ArrowInputFile::ReadAt(int64_t position, int64_t nbytes, uint8_t* out) {
- int64_t bytes_read = 0;
- PARQUET_THROW_NOT_OK(file_->ReadAt(position, nbytes, &bytes_read, out));
- return bytes_read;
-}
-
-ArrowOutputStream::ArrowOutputStream(
- const std::shared_ptr<::arrow::io::OutputStream> file)
- : file_(file) {}
-
-::arrow::io::FileInterface* ArrowOutputStream::file_interface() { return file_.get(); }
-
-// Copy bytes into the output stream
-void ArrowOutputStream::Write(const uint8_t* data, int64_t length) {
- PARQUET_THROW_NOT_OK(file_->Write(data, length));
-}
-
-// ----------------------------------------------------------------------
-// InMemoryInputStream
-
-InMemoryInputStream::InMemoryInputStream(const std::shared_ptr<Buffer>& buffer)
- : buffer_(buffer), offset_(0) {
- len_ = buffer_->size();
-}
-
-InMemoryInputStream::InMemoryInputStream(RandomAccessSource* source, int64_t start,
- int64_t num_bytes)
- : offset_(0) {
- buffer_ = source->ReadAt(start, num_bytes);
- if (buffer_->size() < num_bytes) {
- throw ParquetException("Unable to read column chunk data");
- }
- len_ = buffer_->size();
-}
-
-const uint8_t* InMemoryInputStream::Peek(int64_t num_to_peek, int64_t* num_bytes) {
- *num_bytes = std::min(static_cast<int64_t>(num_to_peek), len_ - offset_);
- return buffer_->data() + offset_;
-}
-
-const uint8_t* InMemoryInputStream::Read(int64_t num_to_read, int64_t* num_bytes) {
- const uint8_t* result = Peek(num_to_read, num_bytes);
- offset_ += *num_bytes;
- return result;
-}
-
-void InMemoryInputStream::Advance(int64_t num_bytes) { offset_ += num_bytes; }
-
-// ----------------------------------------------------------------------
-// In-memory output stream
-
-InMemoryOutputStream::InMemoryOutputStream(MemoryPool* pool, int64_t initial_capacity)
- : size_(0), capacity_(initial_capacity) {
- if (initial_capacity == 0) {
- initial_capacity = kInMemoryDefaultCapacity;
- }
- buffer_ = AllocateBuffer(pool, initial_capacity);
-}
-
-InMemoryOutputStream::~InMemoryOutputStream() {}
-
-uint8_t* InMemoryOutputStream::Head() { return buffer_->mutable_data() + size_; }
-
-void InMemoryOutputStream::Write(const uint8_t* data, int64_t length) {
- if (size_ + length > capacity_) {
- int64_t new_capacity = capacity_ * 2;
- while (new_capacity < size_ + length) {
- new_capacity *= 2;
- }
- PARQUET_THROW_NOT_OK(buffer_->Resize(new_capacity));
- capacity_ = new_capacity;
- }
- memcpy(Head(), data, length);
- size_ += length;
-}
-
-int64_t InMemoryOutputStream::Tell() { return size_; }
-
-std::shared_ptr<Buffer> InMemoryOutputStream::GetBuffer() {
- PARQUET_THROW_NOT_OK(buffer_->Resize(size_));
- std::shared_ptr<Buffer> result = buffer_;
- buffer_ = nullptr;
- return result;
-}
-
-// ----------------------------------------------------------------------
-// BufferedInputStream
-
-BufferedInputStream::BufferedInputStream(MemoryPool* pool, int64_t buffer_size,
- RandomAccessSource* source, int64_t start,
- int64_t num_bytes)
- : source_(source), stream_offset_(start), stream_end_(start + num_bytes) {
- buffer_ = AllocateBuffer(pool, buffer_size);
- buffer_size_ = buffer_->size();
- // Required to force a lazy read
- buffer_offset_ = buffer_size_;
-}
-
-const uint8_t* BufferedInputStream::Peek(int64_t num_to_peek, int64_t* num_bytes) {
- *num_bytes = std::min(num_to_peek, stream_end_ - stream_offset_);
- // increase the buffer size if needed
- if (*num_bytes > buffer_size_) {
- PARQUET_THROW_NOT_OK(buffer_->Resize(*num_bytes));
- buffer_size_ = buffer_->size();
- DCHECK(buffer_size_ >= *num_bytes);
- }
- // Read more data when buffer has insufficient left or when resized
- if (*num_bytes > (buffer_size_ - buffer_offset_)) {
- buffer_size_ = std::min(buffer_size_, stream_end_ - stream_offset_);
- int64_t bytes_read =
- source_->ReadAt(stream_offset_, buffer_size_, buffer_->mutable_data());
- if (bytes_read < *num_bytes) {
- throw ParquetException("Failed reading column data from source");
- }
- buffer_offset_ = 0;
- }
- return buffer_->data() + buffer_offset_;
-}
-
-const uint8_t* BufferedInputStream::Read(int64_t num_to_read, int64_t* num_bytes) {
- const uint8_t* result = Peek(num_to_read, num_bytes);
- stream_offset_ += *num_bytes;
- buffer_offset_ += *num_bytes;
- return result;
-}
-
-void BufferedInputStream::Advance(int64_t num_bytes) {
- stream_offset_ += num_bytes;
- buffer_offset_ += num_bytes;
-}
-
-std::shared_ptr<ResizableBuffer> AllocateBuffer(MemoryPool* pool, int64_t size) {
- std::shared_ptr<ResizableBuffer> result;
- PARQUET_THROW_NOT_OK(arrow::AllocateResizableBuffer(pool, size, &result));
- return result;
-}
-
-} // namespace parquet
diff --git a/src/parquet/util/memory.h b/src/parquet/util/memory.h
deleted file mode 100644
index 088f86f..0000000
--- a/src/parquet/util/memory.h
+++ /dev/null
@@ -1,445 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#ifndef PARQUET_UTIL_MEMORY_H
-#define PARQUET_UTIL_MEMORY_H
-
-#include <atomic>
-#include <cstdint>
-#include <cstdlib>
-#include <cstring>
-#include <memory>
-#include <string>
-#include <vector>
-
-#include "arrow/buffer.h"
-#include "arrow/io/interfaces.h"
-#include "arrow/io/memory.h"
-#include "arrow/memory_pool.h"
-#include "arrow/status.h"
-#include "arrow/util/compression.h"
-
-#include "parquet/exception.h"
-#include "parquet/types.h"
-#include "parquet/util/macros.h"
-#include "parquet/util/visibility.h"
-
-namespace parquet {
-
-static inline std::unique_ptr<::arrow::Codec> GetCodecFromArrow(Compression::type codec) {
- std::unique_ptr<::arrow::Codec> result;
- switch (codec) {
- case Compression::UNCOMPRESSED:
- break;
- case Compression::SNAPPY:
- PARQUET_THROW_NOT_OK(::arrow::Codec::Create(::arrow::Compression::SNAPPY, &result));
- break;
- case Compression::GZIP:
- PARQUET_THROW_NOT_OK(::arrow::Codec::Create(::arrow::Compression::GZIP, &result));
- break;
- case Compression::LZO:
- PARQUET_THROW_NOT_OK(::arrow::Codec::Create(::arrow::Compression::LZO, &result));
- break;
- case Compression::BROTLI:
- PARQUET_THROW_NOT_OK(::arrow::Codec::Create(::arrow::Compression::BROTLI, &result));
- break;
- case Compression::LZ4:
- PARQUET_THROW_NOT_OK(::arrow::Codec::Create(::arrow::Compression::LZ4, &result));
- break;
- case Compression::ZSTD:
- PARQUET_THROW_NOT_OK(::arrow::Codec::Create(::arrow::Compression::ZSTD, &result));
- break;
- default:
- break;
- }
- return result;
-}
-
-static constexpr int64_t kInMemoryDefaultCapacity = 1024;
-
-using Buffer = ::arrow::Buffer;
-using MutableBuffer = ::arrow::MutableBuffer;
-using ResizableBuffer = ::arrow::ResizableBuffer;
-using ResizableBuffer = ::arrow::ResizableBuffer;
-
-template <class T>
-class PARQUET_EXPORT Vector {
- public:
- explicit Vector(int64_t size, ::arrow::MemoryPool* pool);
- void Resize(int64_t new_size);
- void Reserve(int64_t new_capacity);
- void Assign(int64_t size, const T val);
- void Swap(Vector<T>& v);
- inline T& operator[](int64_t i) const { return data_[i]; }
-
- const T* data() const { return data_; }
-
- private:
- std::shared_ptr<ResizableBuffer> buffer_;
- int64_t size_;
- int64_t capacity_;
- T* data_;
-
- DISALLOW_COPY_AND_ASSIGN(Vector);
-};
-
-/// A ChunkedAllocator maintains a list of memory chunks from which it
-/// allocates memory in response to Allocate() calls; Chunks stay around for
-/// the lifetime of the allocator or until they are passed on to another
-/// allocator.
-//
-/// An Allocate() call will attempt to allocate memory from the chunk that was most
-/// recently added; if that chunk doesn't have enough memory to
-/// satisfy the allocation request, the free chunks are searched for one that is
-/// big enough otherwise a new chunk is added to the list.
-/// The current_chunk_idx_ always points to the last chunk with allocated memory.
-/// In order to keep allocation overhead low, chunk sizes double with each new one
-/// added, until they hit a maximum size.
-//
-/// Example:
-/// ChunkedAllocator* p = new ChunkedAllocator();
-/// for (int i = 0; i < 1024; ++i) {
-/// returns 8-byte aligned memory (effectively 24 bytes):
-/// .. = p->Allocate(17);
-/// }
-/// at this point, 17K have been handed out in response to Allocate() calls and
-/// 28K of chunks have been allocated (chunk sizes: 4K, 8K, 16K)
-/// We track total and peak allocated bytes. At this point they would be the same:
-/// 28k bytes. A call to Clear will return the allocated memory so
-/// total_allocate_bytes_
-/// becomes 0 while peak_allocate_bytes_ remains at 28k.
-/// p->Clear();
-/// the entire 1st chunk is returned:
-/// .. = p->Allocate(4 * 1024);
-/// 4K of the 2nd chunk are returned:
-/// .. = p->Allocate(4 * 1024);
-/// a new 20K chunk is created
-/// .. = p->Allocate(20 * 1024);
-//
-/// ChunkedAllocator* p2 = new ChunkedAllocator();
-/// the new ChunkedAllocator receives all chunks containing data from p
-/// p2->AcquireData(p, false);
-/// At this point p.total_allocated_bytes_ would be 0 while p.peak_allocated_bytes_
-/// remains unchanged.
-/// The one remaining (empty) chunk is released:
-/// delete p;
-
-class PARQUET_EXPORT ChunkedAllocator {
- public:
- explicit ChunkedAllocator(::arrow::MemoryPool* pool = ::arrow::default_memory_pool());
-
- /// Frees all chunks of memory and subtracts the total allocated bytes
- /// from the registered limits.
- ~ChunkedAllocator();
-
- /// Allocates 8-byte aligned section of memory of 'size' bytes at the end
- /// of the the current chunk. Creates a new chunk if there aren't any chunks
- /// with enough capacity.
- uint8_t* Allocate(int size);
-
- /// Returns 'byte_size' to the current chunk back to the mem pool. This can
- /// only be used to return either all or part of the previous allocation returned
- /// by Allocate().
- void ReturnPartialAllocation(int byte_size);
-
- /// Makes all allocated chunks available for re-use, but doesn't delete any chunks.
- void Clear();
-
- /// Deletes all allocated chunks. FreeAll() or AcquireData() must be called for
- /// each mem pool
- void FreeAll();
-
- /// Absorb all chunks that hold data from src. If keep_current is true, let src hold on
- /// to its last allocated chunk that contains data.
- /// All offsets handed out by calls to GetCurrentOffset() for 'src' become invalid.
- void AcquireData(ChunkedAllocator* src, bool keep_current);
-
- std::string DebugString();
-
- int64_t total_allocated_bytes() const { return total_allocated_bytes_; }
- int64_t peak_allocated_bytes() const { return peak_allocated_bytes_; }
- int64_t total_reserved_bytes() const { return total_reserved_bytes_; }
-
- /// Return sum of chunk_sizes_.
- int64_t GetTotalChunkSizes() const;
-
- private:
- friend class ChunkedAllocatorTest;
- static const int INITIAL_CHUNK_SIZE = 4 * 1024;
-
- /// The maximum size of chunk that should be allocated. Allocations larger than this
- /// size will get their own individual chunk.
- static const int MAX_CHUNK_SIZE = 1024 * 1024;
-
- struct ChunkInfo {
- uint8_t* data; // Owned by the ChunkInfo.
- int64_t size; // in bytes
-
- /// bytes allocated via Allocate() in this chunk
- int64_t allocated_bytes;
-
- explicit ChunkInfo(int64_t size, uint8_t* buf);
-
- ChunkInfo() : data(nullptr), size(0), allocated_bytes(0) {}
- };
-
- /// chunk from which we served the last Allocate() call;
- /// always points to the last chunk that contains allocated data;
- /// chunks 0..current_chunk_idx_ are guaranteed to contain data
- /// (chunks_[i].allocated_bytes > 0 for i: 0..current_chunk_idx_);
- /// -1 if no chunks present
- int current_chunk_idx_;
-
- /// The size of the next chunk to allocate.
- int64_t next_chunk_size_;
-
- /// sum of allocated_bytes_
- int64_t total_allocated_bytes_;
-
- /// Maximum number of bytes allocated from this pool at one time.
- int64_t peak_allocated_bytes_;
-
- /// sum of all bytes allocated in chunks_
- int64_t total_reserved_bytes_;
-
- std::vector<ChunkInfo> chunks_;
-
- ::arrow::MemoryPool* pool_;
-
- /// Find or allocated a chunk with at least min_size spare capacity and update
- /// current_chunk_idx_. Also updates chunks_, chunk_sizes_ and allocated_bytes_
- /// if a new chunk needs to be created.
- bool FindChunk(int64_t min_size);
-
- /// Check integrity of the supporting data structures; always returns true but DCHECKs
- /// all invariants.
- /// If 'current_chunk_empty' is false, checks that the current chunk contains data.
- bool CheckIntegrity(bool current_chunk_empty);
-
- /// Return offset to unoccpied space in current chunk.
- int GetFreeOffset() const {
- if (current_chunk_idx_ == -1) return 0;
- return static_cast<int>(chunks_[current_chunk_idx_].allocated_bytes);
- }
-
- template <bool CHECK_LIMIT_FIRST>
- uint8_t* Allocate(int size);
-};
-
-// File input and output interfaces that translate arrow::Status to exceptions
-
-class PARQUET_EXPORT FileInterface {
- public:
- virtual ~FileInterface() = default;
-
- // Close the file
- virtual void Close() = 0;
-
- // Return the current position in the file relative to the start
- virtual int64_t Tell() = 0;
-};
-
-/// It is the responsibility of implementations to mind threadsafety of shared
-/// resources
-class PARQUET_EXPORT RandomAccessSource : virtual public FileInterface {
- public:
- virtual ~RandomAccessSource() = default;
-
- virtual int64_t Size() const = 0;
-
- // Returns bytes read
- virtual int64_t Read(int64_t nbytes, uint8_t* out) = 0;
-
- virtual std::shared_ptr<Buffer> Read(int64_t nbytes) = 0;
-
- virtual std::shared_ptr<Buffer> ReadAt(int64_t position, int64_t nbytes) = 0;
-
- /// Returns bytes read
- virtual int64_t ReadAt(int64_t position, int64_t nbytes, uint8_t* out) = 0;
-};
-
-class PARQUET_EXPORT OutputStream : virtual public FileInterface {
- public:
- virtual ~OutputStream() = default;
-
- // Copy bytes into the output stream
- virtual void Write(const uint8_t* data, int64_t length) = 0;
-};
-
-class PARQUET_EXPORT ArrowFileMethods : virtual public FileInterface {
- public:
- // No-op. Closing the file is the responsibility of the owner of the handle
- void Close() override;
-
- int64_t Tell() override;
-
- protected:
- virtual ::arrow::io::FileInterface* file_interface() = 0;
-};
-
-/// This interface depends on the threadsafety of the underlying Arrow file interface
-class PARQUET_EXPORT ArrowInputFile : public ArrowFileMethods, public RandomAccessSource {
- public:
- explicit ArrowInputFile(
- const std::shared_ptr<::arrow::io::ReadableFileInterface>& file);
-
- int64_t Size() const override;
-
- // Returns bytes read
- int64_t Read(int64_t nbytes, uint8_t* out) override;
-
- std::shared_ptr<Buffer> Read(int64_t nbytes) override;
-
- std::shared_ptr<Buffer> ReadAt(int64_t position, int64_t nbytes) override;
-
- /// Returns bytes read
- int64_t ReadAt(int64_t position, int64_t nbytes, uint8_t* out) override;
-
- std::shared_ptr<::arrow::io::ReadableFileInterface> file() const { return file_; }
-
- // Diamond inheritance
- using ArrowFileMethods::Close;
- using ArrowFileMethods::Tell;
-
- private:
- ::arrow::io::FileInterface* file_interface() override;
- std::shared_ptr<::arrow::io::ReadableFileInterface> file_;
-};
-
-class PARQUET_EXPORT ArrowOutputStream : public ArrowFileMethods, public OutputStream {
- public:
- explicit ArrowOutputStream(const std::shared_ptr<::arrow::io::OutputStream> file);
-
- // Copy bytes into the output stream
- void Write(const uint8_t* data, int64_t length) override;
-
- std::shared_ptr<::arrow::io::OutputStream> file() { return file_; }
-
- // Diamond inheritance
- using ArrowFileMethods::Close;
- using ArrowFileMethods::Tell;
-
- private:
- ::arrow::io::FileInterface* file_interface() override;
- std::shared_ptr<::arrow::io::OutputStream> file_;
-};
-
-class PARQUET_EXPORT InMemoryOutputStream : public OutputStream {
- public:
- explicit InMemoryOutputStream(
- ::arrow::MemoryPool* pool = ::arrow::default_memory_pool(),
- int64_t initial_capacity = kInMemoryDefaultCapacity);
-
- virtual ~InMemoryOutputStream();
-
- // Close is currently a no-op with the in-memory stream
- virtual void Close() {}
-
- virtual int64_t Tell();
-
- virtual void Write(const uint8_t* data, int64_t length);
-
- // Clears the stream
- void Clear() { size_ = 0; }
-
- // Get pointer to the underlying buffer
- const Buffer& GetBufferRef() const { return *buffer_; }
-
- // Return complete stream as Buffer
- std::shared_ptr<Buffer> GetBuffer();
-
- private:
- // Mutable pointer to the current write position in the stream
- uint8_t* Head();
-
- std::shared_ptr<ResizableBuffer> buffer_;
- int64_t size_;
- int64_t capacity_;
-
- DISALLOW_COPY_AND_ASSIGN(InMemoryOutputStream);
-};
-
-// ----------------------------------------------------------------------
-// Streaming input interfaces
-
-// Interface for the column reader to get the bytes. The interface is a stream
-// interface, meaning the bytes in order and once a byte is read, it does not
-// need to be read again.
-class InputStream {
- public:
- // Returns the next 'num_to_peek' without advancing the current position.
- // *num_bytes will contain the number of bytes returned which can only be
- // less than num_to_peek at end of stream cases.
- // Since the position is not advanced, calls to this function are idempotent.
- // The buffer returned to the caller is still owned by the input stream and must
- // stay valid until the next call to Peek() or Read().
- virtual const uint8_t* Peek(int64_t num_to_peek, int64_t* num_bytes) = 0;
-
- // Identical to Peek(), except the current position in the stream is advanced by
- // *num_bytes.
- virtual const uint8_t* Read(int64_t num_to_read, int64_t* num_bytes) = 0;
-
- // Advance the stream without reading
- virtual void Advance(int64_t num_bytes) = 0;
-
- virtual ~InputStream() {}
-
- protected:
- InputStream() {}
-};
-
-// Implementation of an InputStream when all the bytes are in memory.
-class PARQUET_EXPORT InMemoryInputStream : public InputStream {
- public:
- InMemoryInputStream(RandomAccessSource* source, int64_t start, int64_t end);
- explicit InMemoryInputStream(const std::shared_ptr<Buffer>& buffer);
- virtual const uint8_t* Peek(int64_t num_to_peek, int64_t* num_bytes);
- virtual const uint8_t* Read(int64_t num_to_read, int64_t* num_bytes);
-
- virtual void Advance(int64_t num_bytes);
-
- private:
- std::shared_ptr<Buffer> buffer_;
- int64_t len_;
- int64_t offset_;
-};
-
-// Implementation of an InputStream when only some of the bytes are in memory.
-class PARQUET_EXPORT BufferedInputStream : public InputStream {
- public:
- BufferedInputStream(::arrow::MemoryPool* pool, int64_t buffer_size,
- RandomAccessSource* source, int64_t start, int64_t end);
- virtual const uint8_t* Peek(int64_t num_to_peek, int64_t* num_bytes);
- virtual const uint8_t* Read(int64_t num_to_read, int64_t* num_bytes);
-
- virtual void Advance(int64_t num_bytes);
-
- private:
- std::shared_ptr<ResizableBuffer> buffer_;
- RandomAccessSource* source_;
- int64_t stream_offset_;
- int64_t stream_end_;
- int64_t buffer_offset_;
- int64_t buffer_size_;
-};
-
-std::shared_ptr<ResizableBuffer> PARQUET_EXPORT AllocateBuffer(
- ::arrow::MemoryPool* pool = ::arrow::default_memory_pool(), int64_t size = 0);
-
-} // namespace parquet
-
-#endif // PARQUET_UTIL_MEMORY_H
diff --git a/src/parquet/util/schema-util.h b/src/parquet/util/schema-util.h
deleted file mode 100644
index 1c66f67..0000000
--- a/src/parquet/util/schema-util.h
+++ /dev/null
@@ -1,87 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#ifndef PARQUET_SCHEMA_UTIL_H
-#define PARQUET_SCHEMA_UTIL_H
-
-#include <string>
-#include <unordered_set>
-#include <vector>
-
-#include "parquet/exception.h"
-#include "parquet/schema.h"
-#include "parquet/types.h"
-
-using parquet::LogicalType;
-using parquet::ParquetException;
-using parquet::SchemaDescriptor;
-using parquet::schema::GroupNode;
-using parquet::schema::Node;
-using parquet::schema::NodePtr;
-
-inline bool str_endswith_tuple(const std::string& str) {
- if (str.size() >= 6) {
- return str.substr(str.size() - 6, 6) == "_tuple";
- }
- return false;
-}
-
-// Special case mentioned in the format spec:
-// If the name is array or ends in _tuple, this should be a list of struct
-// even for single child elements.
-inline bool HasStructListName(const GroupNode& node) {
- return (node.name() == "array" || str_endswith_tuple(node.name()));
-}
-
-// TODO(itaiin): This aux. function is to be deleted once repeated structs are supported
-inline bool IsSimpleStruct(const Node* node) {
- if (!node->is_group()) return false;
- if (node->is_repeated()) return false;
- if (node->logical_type() == LogicalType::LIST) return false;
- // Special case mentioned in the format spec:
- // If the name is array or ends in _tuple, this should be a list of struct
- // even for single child elements.
- auto group = static_cast<const GroupNode*>(node);
- if (group->field_count() == 1 && HasStructListName(*group)) return false;
-
- return true;
-}
-
-// Coalesce a list of schema fields indices which are the roots of the
-// columns referred by a list of column indices
-inline bool ColumnIndicesToFieldIndices(const SchemaDescriptor& descr,
- const std::vector<int>& column_indices,
- std::vector<int>* out) {
- const GroupNode* group = descr.group_node();
- std::unordered_set<int> already_added;
- out->clear();
- for (auto& column_idx : column_indices) {
- auto field_node = descr.GetColumnRoot(column_idx);
- auto field_idx = group->FieldIndex(*field_node);
- if (field_idx < 0) {
- return false;
- }
- auto insertion = already_added.insert(field_idx);
- if (insertion.second) {
- out->push_back(field_idx);
- }
- }
-
- return true;
-}
-
-#endif // PARQUET_SCHEMA_UTIL_H
diff --git a/src/parquet/util/stopwatch.h b/src/parquet/util/stopwatch.h
deleted file mode 100644
index 68cf792..0000000
--- a/src/parquet/util/stopwatch.h
+++ /dev/null
@@ -1,52 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#ifndef PARQUET_UTIL_STOPWATCH_H
-#define PARQUET_UTIL_STOPWATCH_H
-
-#include <stdio.h>
-#ifndef _MSC_VER
-#include <sys/time.h>
-#endif
-
-#include <ctime>
-#include <iostream>
-
-namespace parquet {
-
-class StopWatch {
- public:
- StopWatch() {}
-
- void Start() { gettimeofday(&start_time, 0); }
-
- // Returns time in nanoseconds.
- uint64_t Stop() {
- struct timeval t_time;
- gettimeofday(&t_time, 0);
-
- return (1000L * 1000L * 1000L * (t_time.tv_sec - start_time.tv_sec) +
- (t_time.tv_usec - start_time.tv_usec));
- }
-
- private:
- struct timeval start_time;
-};
-
-} // namespace parquet
-
-#endif
diff --git a/src/parquet/util/test-common.h b/src/parquet/util/test-common.h
deleted file mode 100644
index 4e95870..0000000
--- a/src/parquet/util/test-common.h
+++ /dev/null
@@ -1,208 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#ifndef PARQUET_UTIL_TEST_COMMON_H
-#define PARQUET_UTIL_TEST_COMMON_H
-
-#include <chrono>
-#include <iostream>
-#include <limits>
-#include <random>
-#include <vector>
-
-#include "parquet/exception.h"
-#include "parquet/types.h"
-
-using std::vector;
-
-namespace parquet {
-
-namespace test {
-
-typedef ::testing::Types<BooleanType, Int32Type, Int64Type, Int96Type, FloatType,
- DoubleType, ByteArrayType, FLBAType>
- ParquetTypes;
-
-class ParquetTestException : public parquet::ParquetException {
- using ParquetException::ParquetException;
-};
-
-const char* get_data_dir() {
- const auto result = std::getenv("PARQUET_TEST_DATA");
- if (!result || !result[0]) {
- throw ParquetTestException(
- "Please point the PARQUET_TEST_DATA environment "
- "variable to the test data directory");
- }
- return result;
-}
-
-template <typename T>
-static inline void assert_vector_equal(const vector<T>& left, const vector<T>& right) {
- ASSERT_EQ(left.size(), right.size());
-
- for (size_t i = 0; i < left.size(); ++i) {
- ASSERT_EQ(left[i], right[i]) << i;
- }
-}
-
-template <typename T>
-static inline bool vector_equal(const vector<T>& left, const vector<T>& right) {
- if (left.size() != right.size()) {
- return false;
- }
-
- for (size_t i = 0; i < left.size(); ++i) {
- if (left[i] != right[i]) {
- std::cerr << "index " << i << " left was " << left[i] << " right was " << right[i]
- << std::endl;
- return false;
- }
- }
-
- return true;
-}
-
-template <typename T>
-static vector<T> slice(const vector<T>& values, int start, int end) {
- if (end < start) {
- return vector<T>(0);
- }
-
- vector<T> out(end - start);
- for (int i = start; i < end; ++i) {
- out[i - start] = values[i];
- }
- return out;
-}
-
-static inline vector<bool> flip_coins_seed(int n, double p, uint32_t seed) {
- std::mt19937 gen(seed);
- std::bernoulli_distribution d(p);
-
- vector<bool> draws;
- for (int i = 0; i < n; ++i) {
- draws.push_back(d(gen));
- }
- return draws;
-}
-
-static inline vector<bool> flip_coins(int n, double p) {
- uint64_t seed = std::chrono::high_resolution_clock::now().time_since_epoch().count();
- std::mt19937 gen(static_cast<uint32_t>(seed));
-
- std::bernoulli_distribution d(p);
-
- vector<bool> draws;
- for (int i = 0; i < n; ++i) {
- draws.push_back(d(gen));
- }
- return draws;
-}
-
-void random_bytes(int n, uint32_t seed, std::vector<uint8_t>* out) {
- std::mt19937 gen(seed);
- std::uniform_int_distribution<int> d(0, 255);
-
- for (int i = 0; i < n; ++i) {
- out->push_back(static_cast<uint8_t>(d(gen) & 0xFF));
- }
-}
-
-void random_bools(int n, double p, uint32_t seed, bool* out) {
- std::mt19937 gen(seed);
- std::bernoulli_distribution d(p);
- for (int i = 0; i < n; ++i) {
- out[i] = d(gen);
- }
-}
-
-template <typename T>
-void random_numbers(int n, uint32_t seed, T min_value, T max_value, T* out) {
- std::mt19937 gen(seed);
- std::uniform_int_distribution<T> d(min_value, max_value);
- for (int i = 0; i < n; ++i) {
- out[i] = d(gen);
- }
-}
-
-template <>
-void random_numbers(int n, uint32_t seed, float min_value, float max_value, float* out) {
- std::mt19937 gen(seed);
- std::uniform_real_distribution<float> d(min_value, max_value);
- for (int i = 0; i < n; ++i) {
- out[i] = d(gen);
- }
-}
-
-template <>
-void random_numbers(int n, uint32_t seed, double min_value, double max_value,
- double* out) {
- std::mt19937 gen(seed);
- std::uniform_real_distribution<double> d(min_value, max_value);
- for (int i = 0; i < n; ++i) {
- out[i] = d(gen);
- }
-}
-
-void random_Int96_numbers(int n, uint32_t seed, int32_t min_value, int32_t max_value,
- Int96* out) {
- std::mt19937 gen(seed);
- std::uniform_int_distribution<int32_t> d(min_value, max_value);
- for (int i = 0; i < n; ++i) {
- out[i].value[0] = d(gen);
- out[i].value[1] = d(gen);
- out[i].value[2] = d(gen);
- }
-}
-
-void random_fixed_byte_array(int n, uint32_t seed, uint8_t* buf, int len, FLBA* out) {
- std::mt19937 gen(seed);
- std::uniform_int_distribution<int> d(0, 255);
- for (int i = 0; i < n; ++i) {
- out[i].ptr = buf;
- for (int j = 0; j < len; ++j) {
- buf[j] = static_cast<uint8_t>(d(gen) & 0xFF);
- }
- buf += len;
- }
-}
-
-void random_byte_array(int n, uint32_t seed, uint8_t* buf, ByteArray* out, int min_size,
- int max_size) {
- std::mt19937 gen(seed);
- std::uniform_int_distribution<int> d1(min_size, max_size);
- std::uniform_int_distribution<int> d2(0, 255);
- for (int i = 0; i < n; ++i) {
- int len = d1(gen);
- out[i].len = len;
- out[i].ptr = buf;
- for (int j = 0; j < len; ++j) {
- buf[j] = static_cast<uint8_t>(d2(gen) & 0xFF);
- }
- buf += len;
- }
-}
-
-void random_byte_array(int n, uint32_t seed, uint8_t* buf, ByteArray* out, int max_size) {
- random_byte_array(n, seed, buf, out, 0, max_size);
-}
-
-} // namespace test
-} // namespace parquet
-
-#endif // PARQUET_UTIL_TEST_COMMON_H
diff --git a/src/parquet/util/visibility.h b/src/parquet/util/visibility.h
deleted file mode 100644
index 984fac2..0000000
--- a/src/parquet/util/visibility.h
+++ /dev/null
@@ -1,62 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#ifndef PARQUET_UTIL_VISIBILITY_H
-#define PARQUET_UTIL_VISIBILITY_H
-
-#if defined(_WIN32) || defined(__CYGWIN__)
-#ifdef _MSC_VER
-#pragma warning(push)
-// Disable warning for STL types usage in DLL interface
-// https://web.archive.org/web/20130317015847/http://connect.microsoft.com/VisualStudio/feedback/details/696593/vc-10-vs-2010-basic-string-exports
-#pragma warning(disable : 4275 4251)
-// Disable diamond inheritance warnings
-#pragma warning(disable : 4250)
-// Disable macro redefinition warnings
-#pragma warning(disable : 4005)
-// Disable extern before exported template warnings
-#pragma warning(disable : 4910)
-#endif
-#define PARQUET_EXPORT __declspec(dllexport)
-#define PARQUET_NO_EXPORT
-#else // Not Windows
-#ifndef PARQUET_EXPORT
-#define PARQUET_EXPORT __attribute__((visibility("default")))
-#endif
-#ifndef PARQUET_NO_EXPORT
-#define PARQUET_NO_EXPORT __attribute__((visibility("hidden")))
-#endif
-#endif // Non-Windows
-
-// gcc and clang disagree about how to handle template visibility when you have
-// explicit specializations https://llvm.org/bugs/show_bug.cgi?id=24815
-
-#if defined(__clang__)
-#define PARQUET_EXTERN_TEMPLATE extern template class PARQUET_EXPORT
-#else
-#define PARQUET_EXTERN_TEMPLATE extern template class
-#endif
-
-// This is a complicated topic, some reading on it:
-// http://www.codesynthesis.com/~boris/blog/2010/01/18/dll-export-cxx-templates/
-#ifdef _MSC_VER
-#define PARQUET_TEMPLATE_EXPORT PARQUET_EXPORT
-#else
-#define PARQUET_TEMPLATE_EXPORT
-#endif
-
-#endif // PARQUET_UTIL_VISIBILITY_H
diff --git a/src/parquet/util/windows_compatibility.h b/src/parquet/util/windows_compatibility.h
deleted file mode 100644
index 899590a..0000000
--- a/src/parquet/util/windows_compatibility.h
+++ /dev/null
@@ -1,37 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#ifndef PARQUET_UTIL_WINDOWS_COMPATIBILITY
-#define PARQUET_UTIL_WINDOWS_COMPATIBILITY
-
-#ifdef _WIN32
-
-// Windows defines min and max macros that mess up std::min/max
-#ifndef NOMINMAX
-#define NOMINMAX
-#endif
-
-#include <windows.h>
-#include <winsock2.h>
-
-#ifdef OPTIONAL
-#undef OPTIONAL
-#endif
-
-#endif // _WIN32
-
-#endif // PARQUET_UTIL_WINDOWS_COMPATIBILITY
diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt
deleted file mode 100644
index 0705c83..0000000
--- a/tools/CMakeLists.txt
+++ /dev/null
@@ -1,31 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-if (PARQUET_BUILD_EXECUTABLES)
- set(EXECUTABLE_TOOLS
- parquet-dump-schema
- parquet_reader
- parquet-scan)
-
- foreach(TOOL ${EXECUTABLE_TOOLS})
- add_executable(${TOOL} "${TOOL}.cc")
- target_link_libraries(${TOOL} parquet_static)
- # Avoid unsetting RPATH when installing
- set_target_properties(${TOOL} PROPERTIES INSTALL_RPATH_USE_LINK_PATH TRUE)
- install(TARGETS ${TOOL} RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR})
- endforeach(TOOL)
-endif()
diff --git a/tools/parquet-dump-schema.cc b/tools/parquet-dump-schema.cc
deleted file mode 100644
index 1e0239b..0000000
--- a/tools/parquet-dump-schema.cc
+++ /dev/null
@@ -1,36 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#include <iostream>
-
-#include "parquet/api/reader.h"
-#include "parquet/api/schema.h"
-
-int main(int argc, char** argv) {
- std::string filename = argv[1];
-
- try {
- std::unique_ptr<parquet::ParquetFileReader> reader =
- parquet::ParquetFileReader::OpenFile(filename);
- PrintSchema(reader->metadata()->schema()->schema_root().get(), std::cout);
- } catch (const std::exception& e) {
- std::cerr << "Parquet error: " << e.what() << std::endl;
- return -1;
- }
-
- return 0;
-}
diff --git a/tools/parquet-scan.cc b/tools/parquet-scan.cc
deleted file mode 100644
index ab9363b..0000000
--- a/tools/parquet-scan.cc
+++ /dev/null
@@ -1,78 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#include <ctime>
-#include <iostream>
-#include <list>
-#include <memory>
-
-#include "parquet/api/reader.h"
-
-int main(int argc, char** argv) {
- if (argc > 4 || argc < 1) {
- std::cerr << "Usage: parquet-scan [--batch-size=] [--columns=...] <file>"
- << std::endl;
- return -1;
- }
-
- std::string filename;
-
- // Read command-line options
- int batch_size = 256;
- const std::string COLUMNS_PREFIX = "--columns=";
- const std::string BATCH_SIZE_PREFIX = "--batch-size=";
- std::vector<int> columns;
- int num_columns = 0;
-
- char *param, *value;
- for (int i = 1; i < argc; i++) {
- if ((param = std::strstr(argv[i], COLUMNS_PREFIX.c_str()))) {
- value = std::strtok(param + COLUMNS_PREFIX.length(), ",");
- while (value) {
- columns.push_back(std::atoi(value));
- value = std::strtok(nullptr, ",");
- num_columns++;
- }
- } else if ((param = std::strstr(argv[i], BATCH_SIZE_PREFIX.c_str()))) {
- value = std::strtok(param + BATCH_SIZE_PREFIX.length(), " ");
- if (value) {
- batch_size = std::atoi(value);
- }
- } else {
- filename = argv[i];
- }
- }
-
- try {
- double total_time;
- std::clock_t start_time = std::clock();
- std::unique_ptr<parquet::ParquetFileReader> reader =
- parquet::ParquetFileReader::OpenFile(filename);
-
- int64_t total_rows = parquet::ScanFileContents(columns, batch_size, reader.get());
-
- total_time = static_cast<double>(std::clock() - start_time) /
- static_cast<double>(CLOCKS_PER_SEC);
- std::cout << total_rows << " rows scanned in " << total_time << " seconds."
- << std::endl;
- } catch (const std::exception& e) {
- std::cerr << "Parquet error: " << e.what() << std::endl;
- return -1;
- }
-
- return 0;
-}
diff --git a/tools/parquet_reader.cc b/tools/parquet_reader.cc
deleted file mode 100644
index 34bdfc1..0000000
--- a/tools/parquet_reader.cc
+++ /dev/null
@@ -1,79 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#include <iostream>
-#include <list>
-#include <memory>
-
-#include "parquet/api/reader.h"
-
-int main(int argc, char** argv) {
- if (argc > 5 || argc < 2) {
- std::cerr << "Usage: parquet_reader [--only-metadata] [--no-memory-map] [--json]"
- "[--print-key-value-metadata] [--columns=...] <file>"
- << std::endl;
- return -1;
- }
-
- std::string filename;
- bool print_values = true;
- bool print_key_value_metadata = false;
- bool memory_map = true;
- bool format_json = false;
-
- // Read command-line options
- const std::string COLUMNS_PREFIX = "--columns=";
- std::list<int> columns;
-
- char *param, *value;
- for (int i = 1; i < argc; i++) {
- if ((param = std::strstr(argv[i], "--only-metadata"))) {
- print_values = false;
- } else if ((param = std::strstr(argv[i], "--print-key-value-metadata"))) {
- print_key_value_metadata = true;
- } else if ((param = std::strstr(argv[i], "--no-memory-map"))) {
- memory_map = false;
- } else if ((param = std::strstr(argv[i], "--json"))) {
- format_json = true;
- } else if ((param = std::strstr(argv[i], COLUMNS_PREFIX.c_str()))) {
- value = std::strtok(param + COLUMNS_PREFIX.length(), ",");
- while (value) {
- columns.push_back(std::atoi(value));
- value = std::strtok(nullptr, ",");
- }
- } else {
- filename = argv[i];
- }
- }
-
- try {
- std::unique_ptr<parquet::ParquetFileReader> reader =
- parquet::ParquetFileReader::OpenFile(filename, memory_map);
- parquet::ParquetFilePrinter printer(reader.get());
- if (format_json) {
- printer.JSONPrint(std::cout, columns, filename.c_str());
- } else {
- printer.DebugPrint(std::cout, columns, print_values,
- print_key_value_metadata, filename.c_str());
- }
- } catch (const std::exception& e) {
- std::cerr << "Parquet error: " << e.what() << std::endl;
- return -1;
- }
-
- return 0;
-}
