Removed all.
This will allow to start a project from scratch without losing
the contributions' stats and others.
diff --git a/.gitattributes b/.gitattributes
deleted file mode 100644
index fac7bf8..0000000
--- a/.gitattributes
+++ /dev/null
@@ -1,6 +0,0 @@
-r/R/RcppExports.R linguist-generated=true
-r/R/arrowExports.R linguist-generated=true
-r/src/RcppExports.cpp linguist-generated=true
-r/src/arrowExports.cpp linguist-generated=true
-r/man/*.Rd linguist-generated=true
-
diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md
deleted file mode 100644
index 5600dab..0000000
--- a/.github/ISSUE_TEMPLATE/bug_report.md
+++ /dev/null
@@ -1,20 +0,0 @@
----
-name: Bug report
-about: Create a report to help us improve
-title: ''
-labels: bug
-assignees: ''
-
----
-
-**Describe the bug**
-A clear and concise description of what the bug is.
-
-**To Reproduce**
-Steps to reproduce the behavior:
-
-**Expected behavior**
-A clear and concise description of what you expected to happen.
-
-**Additional context**
-Add any other context about the problem here.
diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md
deleted file mode 100644
index d9883dd..0000000
--- a/.github/ISSUE_TEMPLATE/feature_request.md
+++ /dev/null
@@ -1,21 +0,0 @@
----
-name: Feature request
-about: Suggest an idea for this project
-title: ''
-labels: enhancement
-assignees: ''
-
----
-
-**Is your feature request related to a problem or challenge? Please describe what you are trying to do.**
-A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
-(This section helps Arrow developers understand the context and *why* for this feature, in addition to the *what*)
-
-**Describe the solution you'd like**
-A clear and concise description of what you want to happen.
-
-**Describe alternatives you've considered**
-A clear and concise description of any alternative solutions or features you've considered.
-
-**Additional context**
-Add any other context or screenshots about the feature request here.
diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md
deleted file mode 100644
index db170e3..0000000
--- a/.github/pull_request_template.md
+++ /dev/null
@@ -1,31 +0,0 @@
-# Which issue does this PR close?
-
-<!---
-We generally require a GitHub issue to be filed for all bug fixes and enhancements and this helps us generate change logs for our releases. You can link an issue to this PR using the GitHub syntax. For example `Closes #123` indicates that this PR will close issue #123.
--->
-
-Closes #.
-
-# Rationale for this change
-
- <!---
- Why are you proposing this change? If this is already explained clearly in the issue then this section is not needed.
- Explaining clearly why changes are proposed helps reviewers understand your changes and offer better suggestions for fixes.
--->
-
-# What changes are included in this PR?
-
-<!---
-There is no need to duplicate the description in the issue here but it is sometimes worth providing a summary of the individual changes in this PR.
--->
-
-# Are there any user-facing changes?
-
-
-<!---
-If there are user-facing changes then we may require documentation to be updated before approving the PR.
--->
-
-<!---
-If there are any breaking changes to public APIs, please add the `breaking change` label.
--->
diff --git a/.github/workflows/cancel.yml b/.github/workflows/cancel.yml
deleted file mode 100644
index b4fb904..0000000
--- a/.github/workflows/cancel.yml
+++ /dev/null
@@ -1,53 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-name: Cancel stale runs
-
-on:
- workflow_run:
- # The name of another workflow (whichever one) that always runs on PRs
- workflows: ['Dev']
- types: ['requested']
-
-jobs:
- cancel-stale-workflow-runs:
- name: "Cancel stale workflow runs"
- runs-on: ubuntu-latest
- steps:
- # Unfortunately, we need to define a separate cancellation step for
- # each workflow where we want to cancel stale runs.
- - uses: potiuk/cancel-workflow-runs@master
- name: "Cancel stale Dev runs"
- with:
- cancelMode: allDuplicates
- token: ${{ secrets.GITHUB_TOKEN }}
- workflowFileName: dev.yml
- skipEventTypes: '["push", "schedule"]'
- - uses: potiuk/cancel-workflow-runs@master
- name: "Cancel stale Integration runs"
- with:
- cancelMode: allDuplicates
- token: ${{ secrets.GITHUB_TOKEN }}
- workflowFileName: integration.yml
- skipEventTypes: '["push", "schedule"]'
- - uses: potiuk/cancel-workflow-runs@master
- name: "Cancel stale Rust runs"
- with:
- cancelMode: allDuplicates
- token: ${{ secrets.GITHUB_TOKEN }}
- workflowFileName: rust.yml
- skipEventTypes: '["push", "schedule"]'
diff --git a/.github/workflows/comment_bot.yml b/.github/workflows/comment_bot.yml
deleted file mode 100644
index 6ca0953..0000000
--- a/.github/workflows/comment_bot.yml
+++ /dev/null
@@ -1,72 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-name: Comment Bot
-
-on:
- # TODO(kszucs): support pull_request_review_comment
- issue_comment:
- types:
- - created
- - edited
-
-jobs:
- crossbow:
- name: Listen!
- if: startsWith(github.event.comment.body, '@github-actions crossbow')
- runs-on: ubuntu-latest
- steps:
- - name: Checkout Arrow
- uses: actions/checkout@v2
- with:
- repository: apache/arrow
- - name: Set up Python
- uses: actions/setup-python@v2
- with:
- python-version: 3.8
- - name: Install Archery and Crossbow dependencies
- run: pip install -e dev/archery[bot]
- - name: Handle Github comment event
- env:
- ARROW_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- CROSSBOW_GITHUB_TOKEN: ${{ secrets.CROSSBOW_GITHUB_TOKEN }}
- run: |
- archery trigger-bot \
- --event-name ${{ github.event_name }} \
- --event-payload ${{ github.event_path }}
-
- rebase:
- name: "Rebase"
- if: startsWith(github.event.comment.body, '@github-actions rebase')
- runs-on: ubuntu-latest
- steps:
- - uses: actions/checkout@v2
- - uses: r-lib/actions/pr-fetch@master
- with:
- repo-token: ${{ secrets.GITHUB_TOKEN }}
- - name: Rebase on ${{ github.repository }} master
- run: |
- set -ex
- git config user.name "$(git log -1 --pretty=format:%an)"
- git config user.email "$(git log -1 --pretty=format:%ae)"
- git remote add upstream https://github.com/${{ github.repository }}
- git fetch --unshallow upstream master
- git rebase upstream/master
- - uses: r-lib/actions/pr-push@master
- with:
- repo-token: ${{ secrets.GITHUB_TOKEN }}
- args: "--force"
diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml
deleted file mode 100644
index 548f0dd..0000000
--- a/.github/workflows/dev.yml
+++ /dev/null
@@ -1,67 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-name: Dev
-
-on:
- # always trigger
- push:
- pull_request:
-
-env:
- ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }}
- ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }}
-
-jobs:
-
- rat:
- name: Release Audit Tool (RAT)
- runs-on: ubuntu-latest
- steps:
- - name: Checkout Arrow
- uses: actions/checkout@v2
- with:
- repository: apache/arrow
- submodules: true
- fetch-depth: 0
- - name: Checkout Arrow Rust
- uses: actions/checkout@v2
- with:
- path: rust
- fetch-depth: 0
- - name: Setup Python
- uses: actions/setup-python@v1
- with:
- python-version: 3.8
- - name: Setup Archery
- run: pip install -e dev/archery[lint]
- - name: Lint
- run: archery lint --rat
-
- prettier:
- name: Use prettier to check formatting of documents
- runs-on: ubuntu-latest
- steps:
- - uses: actions/checkout@v2
- - uses: actions/setup-node@v2
- with:
- node-version: "14"
- - name: Prettier check
- run: |
- # if you encounter error, try rerun the command below with --write instead of --check
- # and commit the changes
- npx prettier@2.3.0 --check {arrow,arrow-flight,dev,integration-testing,parquet}/**/*.md README.md CODE_OF_CONDUCT.md CONTRIBUTING.md
diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml
deleted file mode 100644
index f5ab81b..0000000
--- a/.github/workflows/rust.yml
+++ /dev/null
@@ -1,232 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-name: Rust
-
-on:
- # always trigger
- push:
- pull_request:
-
-jobs:
-
- # build the library, a compilation step used by multiple steps below
- linux-build-lib:
- name: Build Libraries on AMD64 Rust ${{ matrix.rust }}
- runs-on: ubuntu-latest
- strategy:
- matrix:
- arch: [amd64]
- rust: [stable]
- container:
- image: ${{ matrix.arch }}/rust
- env:
- # Disable full debug symbol generation to speed up CI build and keep memory down
- # "1" means line tables only, which is useful for panic tracebacks.
- RUSTFLAGS: "-C debuginfo=1"
- steps:
- - uses: actions/checkout@v2
- - name: Cache Cargo
- uses: actions/cache@v2
- with:
- # these represent dependencies downloaded by cargo
- # and thus do not depend on the OS, arch nor rust version.
- path: /github/home/.cargo
- key: cargo-cache-
- - name: Cache Rust dependencies
- uses: actions/cache@v2
- with:
- # these represent compiled steps of both dependencies and arrow
- # and thus are specific for a particular OS, arch and rust version.
- path: /github/home/target
- key: ${{ runner.os }}-${{ matrix.arch }}-target-cache-${{ matrix.rust }}-
- - name: Setup Rust toolchain
- run: |
- rustup toolchain install ${{ matrix.rust }}
- rustup default ${{ matrix.rust }}
- rustup component add rustfmt
- - name: Build Workspace
- run: |
- export CARGO_HOME="/github/home/.cargo"
- export CARGO_TARGET_DIR="/github/home/target"
- cargo build
-
- # test the crate
- linux-test:
- name: Test Workspace on AMD64 Rust ${{ matrix.rust }}
- needs: [linux-build-lib]
- runs-on: ubuntu-latest
- strategy:
- matrix:
- arch: [amd64]
- rust: [stable]
- container:
- image: ${{ matrix.arch }}/rust
- env:
- # Disable full debug symbol generation to speed up CI build and keep memory down
- # "1" means line tables only, which is useful for panic tracebacks.
- RUSTFLAGS: "-C debuginfo=1"
- ARROW_TEST_DATA: /__w/arrow-rs/arrow-rs/testing/data
- PARQUET_TEST_DATA: /__w/arrow-rs/arrow-rs/parquet-testing/data
- steps:
- - uses: actions/checkout@v2
- with:
- submodules: true
- - name: Cache Cargo
- uses: actions/cache@v2
- with:
- path: /github/home/.cargo
- # this key equals the ones on `linux-build-lib` for re-use
- key: cargo-cache-
- - name: Cache Rust dependencies
- uses: actions/cache@v2
- with:
- path: /github/home/target
- # this key equals the ones on `linux-build-lib` for re-use
- key: ${{ runner.os }}-${{ matrix.arch }}-target-cache-${{ matrix.rust }}
- - name: Setup Rust toolchain
- run: |
- rustup toolchain install ${{ matrix.rust }}
- rustup default ${{ matrix.rust }}
- rustup component add rustfmt
- - name: Run tests
- run: |
- export CARGO_HOME="/github/home/.cargo"
- export CARGO_TARGET_DIR="/github/home/target"
- # run tests on all workspace members with default feature list
- cargo test
-
- windows-and-macos:
- name: Test on ${{ matrix.os }} Rust ${{ matrix.rust }}
- runs-on: ${{ matrix.os }}
- strategy:
- matrix:
- os: [windows-latest, macos-latest]
- rust: [stable]
- steps:
- - uses: actions/checkout@v2
- with:
- submodules: true
- # TODO: this won't cache anything, which is expensive. Setup this action
- # with a OS-dependent path.
- - name: Setup Rust toolchain
- run: |
- rustup toolchain install ${{ matrix.rust }}
- rustup default ${{ matrix.rust }}
- rustup component add rustfmt
- - name: Run tests
- shell: bash
- run: |
- export ARROW_TEST_DATA=$(pwd)/testing/data
- export PARQUET_TEST_DATA=$(pwd)/parquet-testing/data
- # do not produce debug symbols to keep memory usage down
- export RUSTFLAGS="-C debuginfo=0"
- cargo test
-
- clippy:
- name: Clippy
- needs: [linux-build-lib]
- runs-on: ubuntu-latest
- strategy:
- matrix:
- arch: [amd64]
- rust: [stable]
- container:
- image: ${{ matrix.arch }}/rust
- env:
- # Disable full debug symbol generation to speed up CI build and keep memory down
- # "1" means line tables only, which is useful for panic tracebacks.
- RUSTFLAGS: "-C debuginfo=1"
- steps:
- - uses: actions/checkout@v2
- with:
- submodules: true
- - name: Cache Cargo
- uses: actions/cache@v2
- with:
- path: /github/home/.cargo
- # this key equals the ones on `linux-build-lib` for re-use
- key: cargo-cache-
- - name: Cache Rust dependencies
- uses: actions/cache@v2
- with:
- path: /github/home/target
- # this key equals the ones on `linux-build-lib` for re-use
- key: ${{ runner.os }}-${{ matrix.arch }}-target-cache-${{ matrix.rust }}
- - name: Setup Rust toolchain
- run: |
- rustup toolchain install ${{ matrix.rust }}
- rustup default ${{ matrix.rust }}
- rustup component add rustfmt clippy
- - name: Run clippy
- run: |
- export CARGO_HOME="/github/home/.cargo"
- export CARGO_TARGET_DIR="/github/home/target"
- cargo clippy --all-targets --workspace -- -D warnings -A clippy::redundant_field_names
-
- lint:
- name: Lint
- runs-on: ubuntu-latest
- container:
- image: amd64/rust
- steps:
- - uses: actions/checkout@v2
- - name: Setup toolchain
- run: |
- rustup toolchain install stable
- rustup default stable
- rustup component add rustfmt
- - name: Run
- run: cargo fmt --all -- --check
- coverage:
- name: Coverage
- runs-on: ubuntu-latest
- strategy:
- matrix:
- arch: [amd64]
- rust: [stable]
- steps:
- - uses: actions/checkout@v2
- with:
- submodules: true
- - name: Cache Cargo
- uses: actions/cache@v2
- with:
- path: /home/runner/.cargo
- # this key is not equal because the user is different than on a container (runner vs github)
- key: cargo-coverage-cache-
- - name: Cache Rust dependencies
- uses: actions/cache@v2
- with:
- path: /home/runner/target
- # this key is not equal because coverage uses different compilation flags.
- key: ${{ runner.os }}-${{ matrix.arch }}-target-coverage-cache-${{ matrix.rust }}-
- - name: Run coverage
- run: |
- export CARGO_HOME="/home/runner/.cargo"
- export CARGO_TARGET_DIR="/home/runner/target"
-
- export ARROW_TEST_DATA=$(pwd)/testing/data
- export PARQUET_TEST_DATA=$(pwd)/parquet-testing/data
-
- # 2020-11-15: There is a cargo-tarpaulin regression in 0.17.0
- # see https://github.com/xd009642/tarpaulin/issues/618
- cargo install --version 0.16.0 cargo-tarpaulin
- cargo tarpaulin --out Xml
- - name: Report coverage
- continue-on-error: true
- run: bash <(curl -s https://codecov.io/bash)
diff --git a/.github_changelog_generator b/.github_changelog_generator
deleted file mode 100644
index 52452ed..0000000
--- a/.github_changelog_generator
+++ /dev/null
@@ -1,29 +0,0 @@
-#!/bin/bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-# point to the old changelog in apache/arrow
-front-matter=For older versions, see [apache/arrow/CHANGELOG.md](https://github.com/apache/arrow/blob/master/CHANGELOG.md)\n
-# some issues are just documentation
-add-sections={"documentation":{"prefix":"**Documentation updates:**","labels":["documentation"]}}
-# uncomment to not show PRs. TBD if we shown them or not.
-#pull-requests=false
-# so that the component is shown associated with the issue
-issue-line-labels=arrow,parquet,arrow-flight
-exclude-labels=development-process,invalid
diff --git a/.gitignore b/.gitignore
deleted file mode 100644
index e8d9955..0000000
--- a/.gitignore
+++ /dev/null
@@ -1,8 +0,0 @@
-Cargo.lock
-target
-rusty-tags.vi
-.history
-.flatbuffers/
-.idea/
-.vscode
-venv/*
diff --git a/.gitmodules b/.gitmodules
deleted file mode 100644
index 97da927..0000000
--- a/.gitmodules
+++ /dev/null
@@ -1,6 +0,0 @@
-[submodule "testing"]
- path = testing
- url = https://github.com/apache/arrow-testing
-[submodule "parquet-testing"]
- path = parquet-testing
- url = https://github.com/apache/parquet-testing.git
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
deleted file mode 100644
index 5331a53..0000000
--- a/.pre-commit-config.yaml
+++ /dev/null
@@ -1,69 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-# To use this, install the python package `pre-commit` and
-# run once `pre-commit install`. This will setup a git pre-commit-hook
-# that is executed on each commit and will report the linting problems.
-# To run all hooks on all files use `pre-commit run -a`
-
-repos:
- - repo: local
- hooks:
- - id: rat
- name: Release Audit Tool
- language: system
- entry: bash -c "git archive HEAD --prefix=apache-arrow/ --output=arrow-src.tar && ./dev/release/run-rat.sh arrow-src.tar"
- always_run: true
- pass_filenames: false
- - id: rustfmt
- name: Rust Format
- language: system
- entry: bash -c "cargo +stable fmt --all -- --check"
- files: ^.*\.rs$
- types:
- - file
- - rust
- - id: cmake-format
- name: CMake Format
- language: python
- entry: python run-cmake-format.py
- types: [cmake]
- additional_dependencies:
- - cmake_format==0.5.2
- - id: hadolint
- name: Docker Format
- language: docker_image
- types:
- - dockerfile
- entry: --entrypoint /bin/hadolint hadolint/hadolint:latest -
- exclude: ^dev/.*$
- - repo: git://github.com/pre-commit/pre-commit-hooks
- sha: v1.2.3
- hooks:
- - id: flake8
- name: Python Format
- files: ^(python|dev|integration)/
- types:
- - file
- - python
- - id: flake8
- name: Cython Format
- files: ^python/
- types:
- - file
- - cython
- args: [--config=python/.flake8.cython]
diff --git a/CHANGELOG.md b/CHANGELOG.md
deleted file mode 100644
index c901a4e..0000000
--- a/CHANGELOG.md
+++ /dev/null
@@ -1,78 +0,0 @@
-For older versions, see [apache/arrow/CHANGELOG.md](https://github.com/apache/arrow/blob/master/CHANGELOG.md)
-
-# Changelog
-
-## [4.0.1](https://github.com/apache/arrow-rs/tree/4.0.1) (2021-05-16)
-
-[Full Changelog](https://github.com/apache/arrow-rs/compare/8707fd2b2d17b17bd3e79be0255a18ffaea6914a...4.0.1)
-
-**Implemented enhancements:**
-
-- Improve performance of bound checking option [\#280](https://github.com/apache/arrow-rs/issues/280) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)]
-- extend compute kernel arity to include nullary functions [\#276](https://github.com/apache/arrow-rs/issues/276)
-- Add support for pretty-printing Decimal numbers [\#230](https://github.com/apache/arrow-rs/issues/230) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)]
-- CSV Reader String Dictionary Support [\#228](https://github.com/apache/arrow-rs/issues/228) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)]
-- Add Builder interface for adding Arrays to record batches [\#210](https://github.com/apache/arrow-rs/issues/210) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)]
-- Support auto-vectorization for min/max [\#209](https://github.com/apache/arrow-rs/issues/209) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)]
-- Support LargeUtf8 in sort kernel [\#25](https://github.com/apache/arrow-rs/issues/25) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)]
-
-**Fixed bugs:**
-
-- Rust 1.52 Clippy error [\#266](https://github.com/apache/arrow-rs/issues/266)
-- NaNs can break parquet statistics [\#255](https://github.com/apache/arrow-rs/issues/255) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)]
-- u64::MAX does not roundtrip through parquet [\#254](https://github.com/apache/arrow-rs/issues/254) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)]
-- Integration tests failing to compile \(flatbuffer\) [\#249](https://github.com/apache/arrow-rs/issues/249) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)]
-- Fix compatibility quirks between arrow and parquet structs [\#245](https://github.com/apache/arrow-rs/issues/245) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)]
-- Unable to write non-null Arrow structs to Parquet [\#244](https://github.com/apache/arrow-rs/issues/244) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)]
-- Dev PR / Process \(pull\_request\) Failing on PRs [\#242](https://github.com/apache/arrow-rs/issues/242)
-- schema: missing field `metadata` when deserialize [\#241](https://github.com/apache/arrow-rs/issues/241) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)]
-- Arrow does not compile due to flatbuffers upgrade [\#238](https://github.com/apache/arrow-rs/issues/238) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)]
-- Sort with limit panics for the limit includes some but not all nulls, for large arrays [\#235](https://github.com/apache/arrow-rs/issues/235) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)]
-- Read list field correctly in \<struct\<list\>\> [\#167](https://github.com/apache/arrow-rs/issues/167) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)]
-
-**Documentation updates:**
-
-- Comment out the instructions in the PR template [\#277](https://github.com/apache/arrow-rs/issues/277)
-- Update links to datafusion and ballista in README.md [\#19](https://github.com/apache/arrow-rs/issues/19)
-- Update "repository" in Cargo.toml [\#12](https://github.com/apache/arrow-rs/issues/12)
-- Improve docs for NullArray, new\_null\_array and new\_empty\_array [\#240](https://github.com/apache/arrow-rs/pull/240) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([alamb](https://github.com/alamb))
-
-**Merged pull requests:**
-
-- Remove old release scripts [\#293](https://github.com/apache/arrow-rs/pull/293) ([alamb](https://github.com/alamb))
-- manually bump development version [\#288](https://github.com/apache/arrow-rs/pull/288) ([nevi-me](https://github.com/nevi-me))
-- Fix subtraction underflow when sorting string arrays with many nulls [\#285](https://github.com/apache/arrow-rs/pull/285) ([medwards](https://github.com/medwards))
-- Speed up bound checking in `take` [\#281](https://github.com/apache/arrow-rs/pull/281) ([Dandandan](https://github.com/Dandandan))
-- Update PR template by commenting out instructions [\#278](https://github.com/apache/arrow-rs/pull/278) ([nevi-me](https://github.com/nevi-me))
-- Added Decimal support to pretty-print display utility \(\#230\) [\#273](https://github.com/apache/arrow-rs/pull/273) ([mgill25](https://github.com/mgill25))
-- Fix null struct and list roundtrip [\#270](https://github.com/apache/arrow-rs/pull/270) ([nevi-me](https://github.com/nevi-me))
-- 1.52 clippy fixes [\#267](https://github.com/apache/arrow-rs/pull/267) ([nevi-me](https://github.com/nevi-me))
-- Fix typo in csv/reader.rs [\#265](https://github.com/apache/arrow-rs/pull/265) ([domoritz](https://github.com/domoritz))
-- Fix empty Schema::metadata deserialization error [\#260](https://github.com/apache/arrow-rs/pull/260) ([hulunbier](https://github.com/hulunbier))
-- update datafusion and ballista doc links [\#259](https://github.com/apache/arrow-rs/pull/259) ([Jimexist](https://github.com/Jimexist))
-- support full u32 and u64 roundtrip through parquet [\#258](https://github.com/apache/arrow-rs/pull/258) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([crepererum](https://github.com/crepererum))
-- fix NaN handling in parquet statistics [\#256](https://github.com/apache/arrow-rs/pull/256) ([crepererum](https://github.com/crepererum))
-- \[MINOR\] Added env to run rust in integration. [\#253](https://github.com/apache/arrow-rs/pull/253) ([jorgecarleitao](https://github.com/jorgecarleitao))
-- \[Minor\] Made integration tests always run. [\#248](https://github.com/apache/arrow-rs/pull/248) ([jorgecarleitao](https://github.com/jorgecarleitao))
-- fix parquet max\_definition for non-null structs [\#246](https://github.com/apache/arrow-rs/pull/246) ([nevi-me](https://github.com/nevi-me))
-- Disabled rebase needed until demonstrate working. [\#243](https://github.com/apache/arrow-rs/pull/243) ([jorgecarleitao](https://github.com/jorgecarleitao))
-- pin flatbuffers to 0.8.4 [\#239](https://github.com/apache/arrow-rs/pull/239) ([ritchie46](https://github.com/ritchie46))
-- sort\_primitive result is capped to the min of limit or values.len [\#236](https://github.com/apache/arrow-rs/pull/236) ([medwards](https://github.com/medwards))
-- Read list field correctly [\#234](https://github.com/apache/arrow-rs/pull/234) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([nevi-me](https://github.com/nevi-me))
-- Fix code examples for RecordBatch::try\_from\_iter [\#231](https://github.com/apache/arrow-rs/pull/231) ([alamb](https://github.com/alamb))
-- Support string dictionaries in csv reader \(\#228\) [\#229](https://github.com/apache/arrow-rs/pull/229) ([tustvold](https://github.com/tustvold))
-- support LargeUtf8 in sort kernel [\#26](https://github.com/apache/arrow-rs/pull/26) ([ritchie46](https://github.com/ritchie46))
-- Removed unused files [\#22](https://github.com/apache/arrow-rs/pull/22) ([jorgecarleitao](https://github.com/jorgecarleitao))
-- ARROW-12504: Buffer::from\_slice\_ref set correct capacity [\#18](https://github.com/apache/arrow-rs/pull/18) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([tustvold](https://github.com/tustvold))
-- Add GitHub templates [\#17](https://github.com/apache/arrow-rs/pull/17) ([andygrove](https://github.com/andygrove))
-- ARROW-12493: Add support for writing dictionary arrays to CSV and JSON [\#16](https://github.com/apache/arrow-rs/pull/16) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([tustvold](https://github.com/tustvold))
-- ARROW-12426: \[Rust\] Fix concatentation of arrow dictionaries [\#15](https://github.com/apache/arrow-rs/pull/15) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([tustvold](https://github.com/tustvold))
-- Update repository and homepage urls [\#14](https://github.com/apache/arrow-rs/pull/14) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] [[arrow-flight](https://github.com/apache/arrow-rs/labels/arrow-flight)] [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([Dandandan](https://github.com/Dandandan))
-- Added rebase-needed bot [\#13](https://github.com/apache/arrow-rs/pull/13) ([jorgecarleitao](https://github.com/jorgecarleitao))
-- Added Integration tests against arrow [\#10](https://github.com/apache/arrow-rs/pull/10) ([jorgecarleitao](https://github.com/jorgecarleitao))
-- ARROW-12343: \[Rust\] Support auto-vectorization for min/max [\#9](https://github.com/apache/arrow-rs/pull/9) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([Dandandan](https://github.com/Dandandan))
-- ARROW-12411: \[Rust\] Create RecordBatches from Iterators [\#7](https://github.com/apache/arrow-rs/pull/7) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([alamb](https://github.com/alamb))
-
-
-
-\* *This Changelog was automatically generated by [github_changelog_generator](https://github.com/github-changelog-generator/github-changelog-generator)*
diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md
deleted file mode 100644
index 9a24b9b..0000000
--- a/CODE_OF_CONDUCT.md
+++ /dev/null
@@ -1,24 +0,0 @@
-<!---
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
--->
-
-# Code of Conduct
-
-- [Code of Conduct for The Apache Software Foundation][1]
-
-[1]: https://www.apache.org/foundation/policies/conduct.html
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
deleted file mode 100644
index 18d6a7b..0000000
--- a/CONTRIBUTING.md
+++ /dev/null
@@ -1,77 +0,0 @@
-<!---
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
--->
-
-# How to contribute to Apache Arrow
-
-## Did you find a bug?
-
-The Arrow project uses JIRA as a bug tracker. To report a bug, you'll have
-to first create an account on the
-[Apache Foundation JIRA](https://issues.apache.org/jira/). The JIRA server
-hosts bugs and issues for multiple Apache projects. The JIRA project name
-for Arrow is "ARROW".
-
-To be assigned to an issue, ask an Arrow JIRA admin to go to
-[Arrow Roles](https://issues.apache.org/jira/plugins/servlet/project-config/ARROW/roles),
-click "Add users to a role," and add you to the "Contributor" role. Most
-committers are authorized to do this; if you're a committer and aren't
-able to load that project admin page, have someone else add you to the
-necessary role.
-
-Before you create a new bug entry, we recommend you first
-[search](https://issues.apache.org/jira/projects/ARROW/issues/ARROW-5140?filter=allopenissues)
-among existing Arrow issues.
-
-When you create a new JIRA entry, please don't forget to fill the "Component"
-field. Arrow has many subcomponents and this helps triaging and filtering
-tremendously. Also, we conventionally prefix the issue title with the component
-name in brackets, such as "[C++] Crash in Array::Frobnicate()", so as to make
-lists more easy to navigate, and we'd be grateful if you did the same.
-
-## Did you write a patch that fixes a bug or brings an improvement?
-
-First create a JIRA entry as described above. Then, submit your changes
-as a GitHub Pull Request. We'll ask you to prefix the pull request title
-with the JIRA issue number and the component name in brackets.
-(for example: "ARROW-2345: [C++] Fix crash in Array::Frobnicate()").
-Respecting this convention makes it easier for us to process the backlog
-of submitted Pull Requests.
-
-### Minor Fixes
-
-Any functionality change should have a JIRA opened. For minor changes that
-affect documentation, you do not need to open up a JIRA. Instead you can
-prefix the title of your PR with "MINOR: " if meets the following guidelines:
-
-- Grammar, usage and spelling fixes that affect no more than 2 files
-- Documentation updates affecting no more than 2 files and not more
- than 500 words.
-
-## Do you want to propose a significant new feature or an important refactoring?
-
-We ask that all discussions about major changes in the codebase happen
-publicly on the [arrow-dev mailing-list](https://mail-archives.apache.org/mod_mbox/arrow-dev/).
-
-## Do you have questions about the source code, the build procedure or the development process?
-
-You can also ask on the mailing-list, see above.
-
-## Further information
-
-Please read our [development documentation](https://arrow.apache.org/docs/developers/contributing.html).
diff --git a/Cargo.toml b/Cargo.toml
deleted file mode 100644
index 8f8f309..0000000
--- a/Cargo.toml
+++ /dev/null
@@ -1,23 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-[workspace]
-members = [
- "parquet",
- "parquet_derive",
- "parquet_derive_test",
-]
diff --git a/LICENSE.txt b/LICENSE.txt
deleted file mode 100644
index 4cec07f..0000000
--- a/LICENSE.txt
+++ /dev/null
@@ -1,2220 +0,0 @@
-
- Apache License
- Version 2.0, January 2004
- http://www.apache.org/licenses/
-
- TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
-
- 1. Definitions.
-
- "License" shall mean the terms and conditions for use, reproduction,
- and distribution as defined by Sections 1 through 9 of this document.
-
- "Licensor" shall mean the copyright owner or entity authorized by
- the copyright owner that is granting the License.
-
- "Legal Entity" shall mean the union of the acting entity and all
- other entities that control, are controlled by, or are under common
- control with that entity. For the purposes of this definition,
- "control" means (i) the power, direct or indirect, to cause the
- direction or management of such entity, whether by contract or
- otherwise, or (ii) ownership of fifty percent (50%) or more of the
- outstanding shares, or (iii) beneficial ownership of such entity.
-
- "You" (or "Your") shall mean an individual or Legal Entity
- exercising permissions granted by this License.
-
- "Source" form shall mean the preferred form for making modifications,
- including but not limited to software source code, documentation
- source, and configuration files.
-
- "Object" form shall mean any form resulting from mechanical
- transformation or translation of a Source form, including but
- not limited to compiled object code, generated documentation,
- and conversions to other media types.
-
- "Work" shall mean the work of authorship, whether in Source or
- Object form, made available under the License, as indicated by a
- copyright notice that is included in or attached to the work
- (an example is provided in the Appendix below).
-
- "Derivative Works" shall mean any work, whether in Source or Object
- form, that is based on (or derived from) the Work and for which the
- editorial revisions, annotations, elaborations, or other modifications
- represent, as a whole, an original work of authorship. For the purposes
- of this License, Derivative Works shall not include works that remain
- separable from, or merely link (or bind by name) to the interfaces of,
- the Work and Derivative Works thereof.
-
- "Contribution" shall mean any work of authorship, including
- the original version of the Work and any modifications or additions
- to that Work or Derivative Works thereof, that is intentionally
- submitted to Licensor for inclusion in the Work by the copyright owner
- or by an individual or Legal Entity authorized to submit on behalf of
- the copyright owner. For the purposes of this definition, "submitted"
- means any form of electronic, verbal, or written communication sent
- to the Licensor or its representatives, including but not limited to
- communication on electronic mailing lists, source code control systems,
- and issue tracking systems that are managed by, or on behalf of, the
- Licensor for the purpose of discussing and improving the Work, but
- excluding communication that is conspicuously marked or otherwise
- designated in writing by the copyright owner as "Not a Contribution."
-
- "Contributor" shall mean Licensor and any individual or Legal Entity
- on behalf of whom a Contribution has been received by Licensor and
- subsequently incorporated within the Work.
-
- 2. Grant of Copyright License. Subject to the terms and conditions of
- this License, each Contributor hereby grants to You a perpetual,
- worldwide, non-exclusive, no-charge, royalty-free, irrevocable
- copyright license to reproduce, prepare Derivative Works of,
- publicly display, publicly perform, sublicense, and distribute the
- Work and such Derivative Works in Source or Object form.
-
- 3. Grant of Patent License. Subject to the terms and conditions of
- this License, each Contributor hereby grants to You a perpetual,
- worldwide, non-exclusive, no-charge, royalty-free, irrevocable
- (except as stated in this section) patent license to make, have made,
- use, offer to sell, sell, import, and otherwise transfer the Work,
- where such license applies only to those patent claims licensable
- by such Contributor that are necessarily infringed by their
- Contribution(s) alone or by combination of their Contribution(s)
- with the Work to which such Contribution(s) was submitted. If You
- institute patent litigation against any entity (including a
- cross-claim or counterclaim in a lawsuit) alleging that the Work
- or a Contribution incorporated within the Work constitutes direct
- or contributory patent infringement, then any patent licenses
- granted to You under this License for that Work shall terminate
- as of the date such litigation is filed.
-
- 4. Redistribution. You may reproduce and distribute copies of the
- Work or Derivative Works thereof in any medium, with or without
- modifications, and in Source or Object form, provided that You
- meet the following conditions:
-
- (a) You must give any other recipients of the Work or
- Derivative Works a copy of this License; and
-
- (b) You must cause any modified files to carry prominent notices
- stating that You changed the files; and
-
- (c) You must retain, in the Source form of any Derivative Works
- that You distribute, all copyright, patent, trademark, and
- attribution notices from the Source form of the Work,
- excluding those notices that do not pertain to any part of
- the Derivative Works; and
-
- (d) If the Work includes a "NOTICE" text file as part of its
- distribution, then any Derivative Works that You distribute must
- include a readable copy of the attribution notices contained
- within such NOTICE file, excluding those notices that do not
- pertain to any part of the Derivative Works, in at least one
- of the following places: within a NOTICE text file distributed
- as part of the Derivative Works; within the Source form or
- documentation, if provided along with the Derivative Works; or,
- within a display generated by the Derivative Works, if and
- wherever such third-party notices normally appear. The contents
- of the NOTICE file are for informational purposes only and
- do not modify the License. You may add Your own attribution
- notices within Derivative Works that You distribute, alongside
- or as an addendum to the NOTICE text from the Work, provided
- that such additional attribution notices cannot be construed
- as modifying the License.
-
- You may add Your own copyright statement to Your modifications and
- may provide additional or different license terms and conditions
- for use, reproduction, or distribution of Your modifications, or
- for any such Derivative Works as a whole, provided Your use,
- reproduction, and distribution of the Work otherwise complies with
- the conditions stated in this License.
-
- 5. Submission of Contributions. Unless You explicitly state otherwise,
- any Contribution intentionally submitted for inclusion in the Work
- by You to the Licensor shall be under the terms and conditions of
- this License, without any additional terms or conditions.
- Notwithstanding the above, nothing herein shall supersede or modify
- the terms of any separate license agreement you may have executed
- with Licensor regarding such Contributions.
-
- 6. Trademarks. This License does not grant permission to use the trade
- names, trademarks, service marks, or product names of the Licensor,
- except as required for reasonable and customary use in describing the
- origin of the Work and reproducing the content of the NOTICE file.
-
- 7. Disclaimer of Warranty. Unless required by applicable law or
- agreed to in writing, Licensor provides the Work (and each
- Contributor provides its Contributions) on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
- implied, including, without limitation, any warranties or conditions
- of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
- PARTICULAR PURPOSE. You are solely responsible for determining the
- appropriateness of using or redistributing the Work and assume any
- risks associated with Your exercise of permissions under this License.
-
- 8. Limitation of Liability. In no event and under no legal theory,
- whether in tort (including negligence), contract, or otherwise,
- unless required by applicable law (such as deliberate and grossly
- negligent acts) or agreed to in writing, shall any Contributor be
- liable to You for damages, including any direct, indirect, special,
- incidental, or consequential damages of any character arising as a
- result of this License or out of the use or inability to use the
- Work (including but not limited to damages for loss of goodwill,
- work stoppage, computer failure or malfunction, or any and all
- other commercial damages or losses), even if such Contributor
- has been advised of the possibility of such damages.
-
- 9. Accepting Warranty or Additional Liability. While redistributing
- the Work or Derivative Works thereof, You may choose to offer,
- and charge a fee for, acceptance of support, warranty, indemnity,
- or other liability obligations and/or rights consistent with this
- License. However, in accepting such obligations, You may act only
- on Your own behalf and on Your sole responsibility, not on behalf
- of any other Contributor, and only if You agree to indemnify,
- defend, and hold each Contributor harmless for any liability
- incurred by, or claims asserted against, such Contributor by reason
- of your accepting any such warranty or additional liability.
-
- END OF TERMS AND CONDITIONS
-
- APPENDIX: How to apply the Apache License to your work.
-
- To apply the Apache License to your work, attach the following
- boilerplate notice, with the fields enclosed by brackets "[]"
- replaced with your own identifying information. (Don't include
- the brackets!) The text should be enclosed in the appropriate
- comment syntax for the file format. We also recommend that a
- file or class name and description of purpose be included on the
- same "printed page" as the copyright notice for easier
- identification within third-party archives.
-
- Copyright [yyyy] [name of copyright owner]
-
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
-
---------------------------------------------------------------------------------
-
-src/plasma/fling.cc and src/plasma/fling.h: Apache 2.0
-
-Copyright 2013 Sharvil Nanavati
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
---------------------------------------------------------------------------------
-
-src/plasma/thirdparty/ae: Modified / 3-Clause BSD
-
-Copyright (c) 2006-2010, Salvatore Sanfilippo <antirez at gmail dot com>
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are met:
-
- * Redistributions of source code must retain the above copyright notice,
- this list of conditions and the following disclaimer.
- * Redistributions in binary form must reproduce the above copyright
- notice, this list of conditions and the following disclaimer in the
- documentation and/or other materials provided with the distribution.
- * Neither the name of Redis nor the names of its contributors may be used
- to endorse or promote products derived from this software without
- specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
-LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
-CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
-SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
-INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
-CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
-ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-POSSIBILITY OF SUCH DAMAGE.
-
---------------------------------------------------------------------------------
-
-src/plasma/thirdparty/dlmalloc.c: CC0
-
-This is a version (aka dlmalloc) of malloc/free/realloc written by
-Doug Lea and released to the public domain, as explained at
-http://creativecommons.org/publicdomain/zero/1.0/ Send questions,
-comments, complaints, performance data, etc to dl@cs.oswego.edu
-
---------------------------------------------------------------------------------
-
-src/plasma/common.cc (some portions)
-
-Copyright (c) Austin Appleby (aappleby (AT) gmail)
-
-Some portions of this file are derived from code in the MurmurHash project
-
-All code is released to the public domain. For business purposes, Murmurhash is
-under the MIT license.
-
-https://sites.google.com/site/murmurhash/
-
---------------------------------------------------------------------------------
-
-src/arrow/util (some portions): Apache 2.0, and 3-clause BSD
-
-Some portions of this module are derived from code in the Chromium project,
-copyright (c) Google inc and (c) The Chromium Authors and licensed under the
-Apache 2.0 License or the under the 3-clause BSD license:
-
- Copyright (c) 2013 The Chromium Authors. All rights reserved.
-
- Redistribution and use in source and binary forms, with or without
- modification, are permitted provided that the following conditions are
- met:
-
- * Redistributions of source code must retain the above copyright
- notice, this list of conditions and the following disclaimer.
- * Redistributions in binary form must reproduce the above
- copyright notice, this list of conditions and the following disclaimer
- in the documentation and/or other materials provided with the
- distribution.
- * Neither the name of Google Inc. nor the names of its
- contributors may be used to endorse or promote products derived from
- this software without specific prior written permission.
-
- THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
---------------------------------------------------------------------------------
-
-This project includes code from Daniel Lemire's FrameOfReference project.
-
-https://github.com/lemire/FrameOfReference/blob/6ccaf9e97160f9a3b299e23a8ef739e711ef0c71/src/bpacking.cpp
-
-Copyright: 2013 Daniel Lemire
-Home page: http://lemire.me/en/
-Project page: https://github.com/lemire/FrameOfReference
-License: Apache License Version 2.0 http://www.apache.org/licenses/LICENSE-2.0
-
---------------------------------------------------------------------------------
-
-This project includes code from the TensorFlow project
-
-Copyright 2015 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
---------------------------------------------------------------------------------
-
-This project includes code from the NumPy project.
-
-https://github.com/numpy/numpy/blob/e1f191c46f2eebd6cb892a4bfe14d9dd43a06c4e/numpy/core/src/multiarray/multiarraymodule.c#L2910
-
-https://github.com/numpy/numpy/blob/68fd82271b9ea5a9e50d4e761061dfcca851382a/numpy/core/src/multiarray/datetime.c
-
-Copyright (c) 2005-2017, NumPy Developers.
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are
-met:
-
- * Redistributions of source code must retain the above copyright
- notice, this list of conditions and the following disclaimer.
-
- * Redistributions in binary form must reproduce the above
- copyright notice, this list of conditions and the following
- disclaimer in the documentation and/or other materials provided
- with the distribution.
-
- * Neither the name of the NumPy Developers nor the names of any
- contributors may be used to endorse or promote products derived
- from this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
---------------------------------------------------------------------------------
-
-This project includes code from the Boost project
-
-Boost Software License - Version 1.0 - August 17th, 2003
-
-Permission is hereby granted, free of charge, to any person or organization
-obtaining a copy of the software and accompanying documentation covered by
-this license (the "Software") to use, reproduce, display, distribute,
-execute, and transmit the Software, and to prepare derivative works of the
-Software, and to permit third-parties to whom the Software is furnished to
-do so, all subject to the following:
-
-The copyright notices in the Software and this entire statement, including
-the above license grant, this restriction and the following disclaimer,
-must be included in all copies of the Software, in whole or in part, and
-all derivative works of the Software, unless such copies or derivative
-works are solely in the form of machine-executable object code generated by
-a source language processor.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
-SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
-FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
-ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
-DEALINGS IN THE SOFTWARE.
-
---------------------------------------------------------------------------------
-
-This project includes code from the FlatBuffers project
-
-Copyright 2014 Google Inc.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
---------------------------------------------------------------------------------
-
-This project includes code from the tslib project
-
-Copyright 2015 Microsoft Corporation. All rights reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
---------------------------------------------------------------------------------
-
-This project includes code from the jemalloc project
-
-https://github.com/jemalloc/jemalloc
-
-Copyright (C) 2002-2017 Jason Evans <jasone@canonware.com>.
-All rights reserved.
-Copyright (C) 2007-2012 Mozilla Foundation. All rights reserved.
-Copyright (C) 2009-2017 Facebook, Inc. All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are met:
-1. Redistributions of source code must retain the above copyright notice(s),
- this list of conditions and the following disclaimer.
-2. Redistributions in binary form must reproduce the above copyright notice(s),
- this list of conditions and the following disclaimer in the documentation
- and/or other materials provided with the distribution.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER(S) ``AS IS'' AND ANY EXPRESS
-OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
-MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
-EVENT SHALL THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY DIRECT, INDIRECT,
-INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
-PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
-LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
-OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
-ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
---------------------------------------------------------------------------------
-
-This project includes code from the Go project, BSD 3-clause license + PATENTS
-weak patent termination clause
-(https://github.com/golang/go/blob/master/PATENTS).
-
-Copyright (c) 2009 The Go Authors. All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are
-met:
-
- * Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
- * Redistributions in binary form must reproduce the above
-copyright notice, this list of conditions and the following disclaimer
-in the documentation and/or other materials provided with the
-distribution.
- * Neither the name of Google Inc. nor the names of its
-contributors may be used to endorse or promote products derived from
-this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
---------------------------------------------------------------------------------
-
-This project includes code from the hs2client
-
-https://github.com/cloudera/hs2client
-
-Copyright 2016 Cloudera Inc.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
---------------------------------------------------------------------------------
-
-The script ci/scripts/util_wait_for_it.sh has the following license
-
-Copyright (c) 2016 Giles Hall
-
-Permission is hereby granted, free of charge, to any person obtaining a copy of
-this software and associated documentation files (the "Software"), to deal in
-the Software without restriction, including without limitation the rights to
-use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
-of the Software, and to permit persons to whom the Software is furnished to do
-so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.
-
---------------------------------------------------------------------------------
-
-The script r/configure has the following license (MIT)
-
-Copyright (c) 2017, Jeroen Ooms and Jim Hester
-
-Permission is hereby granted, free of charge, to any person obtaining a copy of
-this software and associated documentation files (the "Software"), to deal in
-the Software without restriction, including without limitation the rights to
-use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
-of the Software, and to permit persons to whom the Software is furnished to do
-so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.
-
---------------------------------------------------------------------------------
-
-cpp/src/arrow/util/logging.cc, cpp/src/arrow/util/logging.h and
-cpp/src/arrow/util/logging-test.cc are adapted from
-Ray Project (https://github.com/ray-project/ray) (Apache 2.0).
-
-Copyright (c) 2016 Ray Project (https://github.com/ray-project/ray)
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
---------------------------------------------------------------------------------
-The files cpp/src/arrow/vendored/datetime/date.h, cpp/src/arrow/vendored/datetime/tz.h,
-cpp/src/arrow/vendored/datetime/tz_private.h, cpp/src/arrow/vendored/datetime/ios.h,
-cpp/src/arrow/vendored/datetime/ios.mm,
-cpp/src/arrow/vendored/datetime/tz.cpp are adapted from
-Howard Hinnant's date library (https://github.com/HowardHinnant/date)
-It is licensed under MIT license.
-
-The MIT License (MIT)
-Copyright (c) 2015, 2016, 2017 Howard Hinnant
-Copyright (c) 2016 Adrian Colomitchi
-Copyright (c) 2017 Florian Dang
-Copyright (c) 2017 Paul Thompson
-Copyright (c) 2018 Tomasz Kamiński
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.
-
---------------------------------------------------------------------------------
-
-The file cpp/src/arrow/util/utf8.h includes code adapted from the page
- https://bjoern.hoehrmann.de/utf-8/decoder/dfa/
-with the following license (MIT)
-
-Copyright (c) 2008-2009 Bjoern Hoehrmann <bjoern@hoehrmann.de>
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.
-
---------------------------------------------------------------------------------
-
-The file cpp/src/arrow/vendored/string_view.hpp has the following license
-
-Boost Software License - Version 1.0 - August 17th, 2003
-
-Permission is hereby granted, free of charge, to any person or organization
-obtaining a copy of the software and accompanying documentation covered by
-this license (the "Software") to use, reproduce, display, distribute,
-execute, and transmit the Software, and to prepare derivative works of the
-Software, and to permit third-parties to whom the Software is furnished to
-do so, all subject to the following:
-
-The copyright notices in the Software and this entire statement, including
-the above license grant, this restriction and the following disclaimer,
-must be included in all copies of the Software, in whole or in part, and
-all derivative works of the Software, unless such copies or derivative
-works are solely in the form of machine-executable object code generated by
-a source language processor.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
-SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
-FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
-ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
-DEALINGS IN THE SOFTWARE.
-
---------------------------------------------------------------------------------
-
-The files in cpp/src/arrow/vendored/xxhash/ have the following license
-(BSD 2-Clause License)
-
-xxHash Library
-Copyright (c) 2012-2014, Yann Collet
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without modification,
-are permitted provided that the following conditions are met:
-
-* Redistributions of source code must retain the above copyright notice, this
- list of conditions and the following disclaimer.
-
-* Redistributions in binary form must reproduce the above copyright notice, this
- list of conditions and the following disclaimer in the documentation and/or
- other materials provided with the distribution.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
-ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
-ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
-(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
-LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
-ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
-SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-You can contact the author at :
-- xxHash homepage: http://www.xxhash.com
-- xxHash source repository : https://github.com/Cyan4973/xxHash
-
---------------------------------------------------------------------------------
-
-The files in cpp/src/arrow/vendored/double-conversion/ have the following license
-(BSD 3-Clause License)
-
-Copyright 2006-2011, the V8 project authors. All rights reserved.
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are
-met:
-
- * Redistributions of source code must retain the above copyright
- notice, this list of conditions and the following disclaimer.
- * Redistributions in binary form must reproduce the above
- copyright notice, this list of conditions and the following
- disclaimer in the documentation and/or other materials provided
- with the distribution.
- * Neither the name of Google Inc. nor the names of its
- contributors may be used to endorse or promote products derived
- from this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
---------------------------------------------------------------------------------
-
-The files in cpp/src/arrow/vendored/uriparser/ have the following license
-(BSD 3-Clause License)
-
-uriparser - RFC 3986 URI parsing library
-
-Copyright (C) 2007, Weijia Song <songweijia@gmail.com>
-Copyright (C) 2007, Sebastian Pipping <sebastian@pipping.org>
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions
-are met:
-
- * Redistributions of source code must retain the above
- copyright notice, this list of conditions and the following
- disclaimer.
-
- * Redistributions in binary form must reproduce the above
- copyright notice, this list of conditions and the following
- disclaimer in the documentation and/or other materials
- provided with the distribution.
-
- * Neither the name of the <ORGANIZATION> nor the names of its
- contributors may be used to endorse or promote products
- derived from this software without specific prior written
- permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
-FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
-COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
-INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
-(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
-SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
-HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
-STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
-ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
-OF THE POSSIBILITY OF SUCH DAMAGE.
-
---------------------------------------------------------------------------------
-
-The files under dev/tasks/conda-recipes have the following license
-
-BSD 3-clause license
-Copyright (c) 2015-2018, conda-forge
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without modification,
-are permitted provided that the following conditions are met:
-
-1. Redistributions of source code must retain the above copyright notice, this
- list of conditions and the following disclaimer.
-
-2. Redistributions in binary form must reproduce the above copyright notice,
- this list of conditions and the following disclaimer in the documentation
- and/or other materials provided with the distribution.
-
-3. Neither the name of the copyright holder nor the names of its contributors
- may be used to endorse or promote products derived from this software without
- specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
-ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
-FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
-DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
-SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
-CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR
-TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
-THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
---------------------------------------------------------------------------------
-
-The files in cpp/src/arrow/vendored/utfcpp/ have the following license
-
-Copyright 2006-2018 Nemanja Trifunovic
-
-Permission is hereby granted, free of charge, to any person or organization
-obtaining a copy of the software and accompanying documentation covered by
-this license (the "Software") to use, reproduce, display, distribute,
-execute, and transmit the Software, and to prepare derivative works of the
-Software, and to permit third-parties to whom the Software is furnished to
-do so, all subject to the following:
-
-The copyright notices in the Software and this entire statement, including
-the above license grant, this restriction and the following disclaimer,
-must be included in all copies of the Software, in whole or in part, and
-all derivative works of the Software, unless such copies or derivative
-works are solely in the form of machine-executable object code generated by
-a source language processor.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
-SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
-FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
-ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
-DEALINGS IN THE SOFTWARE.
-
---------------------------------------------------------------------------------
-
-This project includes code from Apache Kudu.
-
- * cpp/cmake_modules/CompilerInfo.cmake is based on Kudu's cmake_modules/CompilerInfo.cmake
-
-Copyright: 2016 The Apache Software Foundation.
-Home page: https://kudu.apache.org/
-License: http://www.apache.org/licenses/LICENSE-2.0
-
---------------------------------------------------------------------------------
-
-This project includes code from Apache Impala (incubating), formerly
-Impala. The Impala code and rights were donated to the ASF as part of the
-Incubator process after the initial code imports into Apache Parquet.
-
-Copyright: 2012 Cloudera, Inc.
-Copyright: 2016 The Apache Software Foundation.
-Home page: http://impala.apache.org/
-License: http://www.apache.org/licenses/LICENSE-2.0
-
---------------------------------------------------------------------------------
-
-This project includes code from Apache Aurora.
-
-* dev/release/{release,changelog,release-candidate} are based on the scripts from
- Apache Aurora
-
-Copyright: 2016 The Apache Software Foundation.
-Home page: https://aurora.apache.org/
-License: http://www.apache.org/licenses/LICENSE-2.0
-
---------------------------------------------------------------------------------
-
-This project includes code from the Google styleguide.
-
-* cpp/build-support/cpplint.py is based on the scripts from the Google styleguide.
-
-Copyright: 2009 Google Inc. All rights reserved.
-Homepage: https://github.com/google/styleguide
-License: 3-clause BSD
-
---------------------------------------------------------------------------------
-
-This project includes code from Snappy.
-
-* cpp/cmake_modules/{SnappyCMakeLists.txt,SnappyConfig.h} are based on code
- from Google's Snappy project.
-
-Copyright: 2009 Google Inc. All rights reserved.
-Homepage: https://github.com/google/snappy
-License: 3-clause BSD
-
---------------------------------------------------------------------------------
-
-This project includes code from the manylinux project.
-
-* python/manylinux1/scripts/{build_python.sh,python-tag-abi-tag.py,
- requirements.txt} are based on code from the manylinux project.
-
-Copyright: 2016 manylinux
-Homepage: https://github.com/pypa/manylinux
-License: The MIT License (MIT)
-
---------------------------------------------------------------------------------
-
-This project includes code from the cymove project:
-
-* python/pyarrow/includes/common.pxd includes code from the cymove project
-
-The MIT License (MIT)
-Copyright (c) 2019 Omer Ozarslan
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
-IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
-DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
-OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE
-OR OTHER DEALINGS IN THE SOFTWARE.
-
---------------------------------------------------------------------------------
-
-The projects includes code from the Ursabot project under the dev/archery
-directory.
-
-License: BSD 2-Clause
-
-Copyright 2019 RStudio, Inc.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are met:
-
-1. Redistributions of source code must retain the above copyright notice, this
- list of conditions and the following disclaimer.
-
-2. Redistributions in binary form must reproduce the above copyright notice,
- this list of conditions and the following disclaimer in the documentation
- and/or other materials provided with the distribution.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
-ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
-FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
-DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
-SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
-CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
-OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
---------------------------------------------------------------------------------
-
-This project include code from mingw-w64.
-
-* cpp/src/arrow/util/cpu-info.cc has a polyfill for mingw-w64 < 5
-
-Copyright (c) 2009 - 2013 by the mingw-w64 project
-Homepage: https://mingw-w64.org
-License: Zope Public License (ZPL) Version 2.1.
-
----------------------------------------------------------------------------------
-
-This project include code from Google's Asylo project.
-
-* cpp/src/arrow/result.h is based on status_or.h
-
-Copyright (c) Copyright 2017 Asylo authors
-Homepage: https://asylo.dev/
-License: Apache 2.0
-
---------------------------------------------------------------------------------
-
-This project includes code from Google's protobuf project
-
-* cpp/src/arrow/result.h ARROW_ASSIGN_OR_RAISE is based off ASSIGN_OR_RETURN
-
-Copyright 2008 Google Inc. All rights reserved.
-Homepage: https://developers.google.com/protocol-buffers/
-License:
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are
-met:
-
- * Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
- * Redistributions in binary form must reproduce the above
-copyright notice, this list of conditions and the following disclaimer
-in the documentation and/or other materials provided with the
-distribution.
- * Neither the name of Google Inc. nor the names of its
-contributors may be used to endorse or promote products derived from
-this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-Code generated by the Protocol Buffer compiler is owned by the owner
-of the input file used when generating it. This code is not
-standalone and requires a support library to be linked with it. This
-support library is itself covered by the above license.
-
---------------------------------------------------------------------------------
-
-3rdparty dependency LLVM is statically linked in certain binary distributions.
-Additionally some sections of source code have been derived from sources in LLVM
-and have been clearly labeled as such. LLVM has the following license:
-
-==============================================================================
-The LLVM Project is under the Apache License v2.0 with LLVM Exceptions:
-==============================================================================
-
- Apache License
- Version 2.0, January 2004
- http://www.apache.org/licenses/
-
- TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
-
- 1. Definitions.
-
- "License" shall mean the terms and conditions for use, reproduction,
- and distribution as defined by Sections 1 through 9 of this document.
-
- "Licensor" shall mean the copyright owner or entity authorized by
- the copyright owner that is granting the License.
-
- "Legal Entity" shall mean the union of the acting entity and all
- other entities that control, are controlled by, or are under common
- control with that entity. For the purposes of this definition,
- "control" means (i) the power, direct or indirect, to cause the
- direction or management of such entity, whether by contract or
- otherwise, or (ii) ownership of fifty percent (50%) or more of the
- outstanding shares, or (iii) beneficial ownership of such entity.
-
- "You" (or "Your") shall mean an individual or Legal Entity
- exercising permissions granted by this License.
-
- "Source" form shall mean the preferred form for making modifications,
- including but not limited to software source code, documentation
- source, and configuration files.
-
- "Object" form shall mean any form resulting from mechanical
- transformation or translation of a Source form, including but
- not limited to compiled object code, generated documentation,
- and conversions to other media types.
-
- "Work" shall mean the work of authorship, whether in Source or
- Object form, made available under the License, as indicated by a
- copyright notice that is included in or attached to the work
- (an example is provided in the Appendix below).
-
- "Derivative Works" shall mean any work, whether in Source or Object
- form, that is based on (or derived from) the Work and for which the
- editorial revisions, annotations, elaborations, or other modifications
- represent, as a whole, an original work of authorship. For the purposes
- of this License, Derivative Works shall not include works that remain
- separable from, or merely link (or bind by name) to the interfaces of,
- the Work and Derivative Works thereof.
-
- "Contribution" shall mean any work of authorship, including
- the original version of the Work and any modifications or additions
- to that Work or Derivative Works thereof, that is intentionally
- submitted to Licensor for inclusion in the Work by the copyright owner
- or by an individual or Legal Entity authorized to submit on behalf of
- the copyright owner. For the purposes of this definition, "submitted"
- means any form of electronic, verbal, or written communication sent
- to the Licensor or its representatives, including but not limited to
- communication on electronic mailing lists, source code control systems,
- and issue tracking systems that are managed by, or on behalf of, the
- Licensor for the purpose of discussing and improving the Work, but
- excluding communication that is conspicuously marked or otherwise
- designated in writing by the copyright owner as "Not a Contribution."
-
- "Contributor" shall mean Licensor and any individual or Legal Entity
- on behalf of whom a Contribution has been received by Licensor and
- subsequently incorporated within the Work.
-
- 2. Grant of Copyright License. Subject to the terms and conditions of
- this License, each Contributor hereby grants to You a perpetual,
- worldwide, non-exclusive, no-charge, royalty-free, irrevocable
- copyright license to reproduce, prepare Derivative Works of,
- publicly display, publicly perform, sublicense, and distribute the
- Work and such Derivative Works in Source or Object form.
-
- 3. Grant of Patent License. Subject to the terms and conditions of
- this License, each Contributor hereby grants to You a perpetual,
- worldwide, non-exclusive, no-charge, royalty-free, irrevocable
- (except as stated in this section) patent license to make, have made,
- use, offer to sell, sell, import, and otherwise transfer the Work,
- where such license applies only to those patent claims licensable
- by such Contributor that are necessarily infringed by their
- Contribution(s) alone or by combination of their Contribution(s)
- with the Work to which such Contribution(s) was submitted. If You
- institute patent litigation against any entity (including a
- cross-claim or counterclaim in a lawsuit) alleging that the Work
- or a Contribution incorporated within the Work constitutes direct
- or contributory patent infringement, then any patent licenses
- granted to You under this License for that Work shall terminate
- as of the date such litigation is filed.
-
- 4. Redistribution. You may reproduce and distribute copies of the
- Work or Derivative Works thereof in any medium, with or without
- modifications, and in Source or Object form, provided that You
- meet the following conditions:
-
- (a) You must give any other recipients of the Work or
- Derivative Works a copy of this License; and
-
- (b) You must cause any modified files to carry prominent notices
- stating that You changed the files; and
-
- (c) You must retain, in the Source form of any Derivative Works
- that You distribute, all copyright, patent, trademark, and
- attribution notices from the Source form of the Work,
- excluding those notices that do not pertain to any part of
- the Derivative Works; and
-
- (d) If the Work includes a "NOTICE" text file as part of its
- distribution, then any Derivative Works that You distribute must
- include a readable copy of the attribution notices contained
- within such NOTICE file, excluding those notices that do not
- pertain to any part of the Derivative Works, in at least one
- of the following places: within a NOTICE text file distributed
- as part of the Derivative Works; within the Source form or
- documentation, if provided along with the Derivative Works; or,
- within a display generated by the Derivative Works, if and
- wherever such third-party notices normally appear. The contents
- of the NOTICE file are for informational purposes only and
- do not modify the License. You may add Your own attribution
- notices within Derivative Works that You distribute, alongside
- or as an addendum to the NOTICE text from the Work, provided
- that such additional attribution notices cannot be construed
- as modifying the License.
-
- You may add Your own copyright statement to Your modifications and
- may provide additional or different license terms and conditions
- for use, reproduction, or distribution of Your modifications, or
- for any such Derivative Works as a whole, provided Your use,
- reproduction, and distribution of the Work otherwise complies with
- the conditions stated in this License.
-
- 5. Submission of Contributions. Unless You explicitly state otherwise,
- any Contribution intentionally submitted for inclusion in the Work
- by You to the Licensor shall be under the terms and conditions of
- this License, without any additional terms or conditions.
- Notwithstanding the above, nothing herein shall supersede or modify
- the terms of any separate license agreement you may have executed
- with Licensor regarding such Contributions.
-
- 6. Trademarks. This License does not grant permission to use the trade
- names, trademarks, service marks, or product names of the Licensor,
- except as required for reasonable and customary use in describing the
- origin of the Work and reproducing the content of the NOTICE file.
-
- 7. Disclaimer of Warranty. Unless required by applicable law or
- agreed to in writing, Licensor provides the Work (and each
- Contributor provides its Contributions) on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
- implied, including, without limitation, any warranties or conditions
- of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
- PARTICULAR PURPOSE. You are solely responsible for determining the
- appropriateness of using or redistributing the Work and assume any
- risks associated with Your exercise of permissions under this License.
-
- 8. Limitation of Liability. In no event and under no legal theory,
- whether in tort (including negligence), contract, or otherwise,
- unless required by applicable law (such as deliberate and grossly
- negligent acts) or agreed to in writing, shall any Contributor be
- liable to You for damages, including any direct, indirect, special,
- incidental, or consequential damages of any character arising as a
- result of this License or out of the use or inability to use the
- Work (including but not limited to damages for loss of goodwill,
- work stoppage, computer failure or malfunction, or any and all
- other commercial damages or losses), even if such Contributor
- has been advised of the possibility of such damages.
-
- 9. Accepting Warranty or Additional Liability. While redistributing
- the Work or Derivative Works thereof, You may choose to offer,
- and charge a fee for, acceptance of support, warranty, indemnity,
- or other liability obligations and/or rights consistent with this
- License. However, in accepting such obligations, You may act only
- on Your own behalf and on Your sole responsibility, not on behalf
- of any other Contributor, and only if You agree to indemnify,
- defend, and hold each Contributor harmless for any liability
- incurred by, or claims asserted against, such Contributor by reason
- of your accepting any such warranty or additional liability.
-
- END OF TERMS AND CONDITIONS
-
- APPENDIX: How to apply the Apache License to your work.
-
- To apply the Apache License to your work, attach the following
- boilerplate notice, with the fields enclosed by brackets "[]"
- replaced with your own identifying information. (Don't include
- the brackets!) The text should be enclosed in the appropriate
- comment syntax for the file format. We also recommend that a
- file or class name and description of purpose be included on the
- same "printed page" as the copyright notice for easier
- identification within third-party archives.
-
- Copyright [yyyy] [name of copyright owner]
-
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
-
-
----- LLVM Exceptions to the Apache 2.0 License ----
-
-As an exception, if, as a result of your compiling your source code, portions
-of this Software are embedded into an Object form of such source code, you
-may redistribute such embedded portions in such Object form without complying
-with the conditions of Sections 4(a), 4(b) and 4(d) of the License.
-
-In addition, if you combine or link compiled forms of this Software with
-software that is licensed under the GPLv2 ("Combined Software") and if a
-court of competent jurisdiction determines that the patent provision (Section
-3), the indemnity provision (Section 9) or other Section of the License
-conflicts with the conditions of the GPLv2, you may retroactively and
-prospectively choose to deem waived or otherwise exclude such Section(s) of
-the License, but only in their entirety and only with respect to the Combined
-Software.
-
-==============================================================================
-Software from third parties included in the LLVM Project:
-==============================================================================
-The LLVM Project contains third party software which is under different license
-terms. All such code will be identified clearly using at least one of two
-mechanisms:
-1) It will be in a separate directory tree with its own `LICENSE.txt` or
- `LICENSE` file at the top containing the specific license and restrictions
- which apply to that software, or
-2) It will contain specific license and restriction terms at the top of every
- file.
-
---------------------------------------------------------------------------------
-
-3rdparty dependency gRPC is statically linked in certain binary
-distributions, like the python wheels. gRPC has the following license:
-
-Copyright 2014 gRPC authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
---------------------------------------------------------------------------------
-
-3rdparty dependency Apache Thrift is statically linked in certain binary
-distributions, like the python wheels. Apache Thrift has the following license:
-
-Apache Thrift
-Copyright (C) 2006 - 2019, The Apache Software Foundation
-
-This product includes software developed at
-The Apache Software Foundation (http://www.apache.org/).
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
---------------------------------------------------------------------------------
-
-3rdparty dependency Apache ORC is statically linked in certain binary
-distributions, like the python wheels. Apache ORC has the following license:
-
-Apache ORC
-Copyright 2013-2019 The Apache Software Foundation
-
-This product includes software developed by The Apache Software
-Foundation (http://www.apache.org/).
-
-This product includes software developed by Hewlett-Packard:
-(c) Copyright [2014-2015] Hewlett-Packard Development Company, L.P
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
---------------------------------------------------------------------------------
-
-3rdparty dependency zstd is statically linked in certain binary
-distributions, like the python wheels. ZSTD has the following license:
-
-BSD License
-
-For Zstandard software
-
-Copyright (c) 2016-present, Facebook, Inc. All rights reserved.
-
-Redistribution and use in source and binary forms, with or without modification,
-are permitted provided that the following conditions are met:
-
- * Redistributions of source code must retain the above copyright notice, this
- list of conditions and the following disclaimer.
-
- * Redistributions in binary form must reproduce the above copyright notice,
- this list of conditions and the following disclaimer in the documentation
- and/or other materials provided with the distribution.
-
- * Neither the name Facebook nor the names of its contributors may be used to
- endorse or promote products derived from this software without specific
- prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
-ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
-ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
-(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
-LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
-ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
-SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
---------------------------------------------------------------------------------
-
-3rdparty dependency lz4 is statically linked in certain binary
-distributions, like the python wheels. lz4 has the following license:
-
-LZ4 Library
-Copyright (c) 2011-2016, Yann Collet
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without modification,
-are permitted provided that the following conditions are met:
-
-* Redistributions of source code must retain the above copyright notice, this
- list of conditions and the following disclaimer.
-
-* Redistributions in binary form must reproduce the above copyright notice, this
- list of conditions and the following disclaimer in the documentation and/or
- other materials provided with the distribution.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
-ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
-ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
-(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
-LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
-ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
-SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
---------------------------------------------------------------------------------
-
-3rdparty dependency Brotli is statically linked in certain binary
-distributions, like the python wheels. Brotli has the following license:
-
-Copyright (c) 2009, 2010, 2013-2016 by the Brotli Authors.
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in
-all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-THE SOFTWARE.
-
---------------------------------------------------------------------------------
-
-3rdparty dependency rapidjson is statically linked in certain binary
-distributions, like the python wheels. rapidjson and its dependencies have the
-following licenses:
-
-Tencent is pleased to support the open source community by making RapidJSON
-available.
-
-Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip.
-All rights reserved.
-
-If you have downloaded a copy of the RapidJSON binary from Tencent, please note
-that the RapidJSON binary is licensed under the MIT License.
-If you have downloaded a copy of the RapidJSON source code from Tencent, please
-note that RapidJSON source code is licensed under the MIT License, except for
-the third-party components listed below which are subject to different license
-terms. Your integration of RapidJSON into your own projects may require
-compliance with the MIT License, as well as the other licenses applicable to
-the third-party components included within RapidJSON. To avoid the problematic
-JSON license in your own projects, it's sufficient to exclude the
-bin/jsonchecker/ directory, as it's the only code under the JSON license.
-A copy of the MIT License is included in this file.
-
-Other dependencies and licenses:
-
- Open Source Software Licensed Under the BSD License:
- --------------------------------------------------------------------
-
- The msinttypes r29
- Copyright (c) 2006-2013 Alexander Chemeris
- All rights reserved.
-
- Redistribution and use in source and binary forms, with or without
- modification, are permitted provided that the following conditions are met:
-
- * Redistributions of source code must retain the above copyright notice,
- this list of conditions and the following disclaimer.
- * Redistributions in binary form must reproduce the above copyright notice,
- this list of conditions and the following disclaimer in the documentation
- and/or other materials provided with the distribution.
- * Neither the name of copyright holder nor the names of its contributors
- may be used to endorse or promote products derived from this software
- without specific prior written permission.
-
- THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND ANY
- EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
- WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- DISCLAIMED. IN NO EVENT SHALL THE REGENTS AND CONTRIBUTORS BE LIABLE FOR
- ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
- SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
- CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
- DAMAGE.
-
- Open Source Software Licensed Under the JSON License:
- --------------------------------------------------------------------
-
- json.org
- Copyright (c) 2002 JSON.org
- All Rights Reserved.
-
- JSON_checker
- Copyright (c) 2002 JSON.org
- All Rights Reserved.
-
-
- Terms of the JSON License:
- ---------------------------------------------------
-
- Permission is hereby granted, free of charge, to any person obtaining a
- copy of this software and associated documentation files (the "Software"),
- to deal in the Software without restriction, including without limitation
- the rights to use, copy, modify, merge, publish, distribute, sublicense,
- and/or sell copies of the Software, and to permit persons to whom the
- Software is furnished to do so, subject to the following conditions:
-
- The above copyright notice and this permission notice shall be included in
- all copies or substantial portions of the Software.
-
- The Software shall be used for Good, not Evil.
-
- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
- DEALINGS IN THE SOFTWARE.
-
-
- Terms of the MIT License:
- --------------------------------------------------------------------
-
- Permission is hereby granted, free of charge, to any person obtaining a
- copy of this software and associated documentation files (the "Software"),
- to deal in the Software without restriction, including without limitation
- the rights to use, copy, modify, merge, publish, distribute, sublicense,
- and/or sell copies of the Software, and to permit persons to whom the
- Software is furnished to do so, subject to the following conditions:
-
- The above copyright notice and this permission notice shall be included
- in all copies or substantial portions of the Software.
-
- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
- DEALINGS IN THE SOFTWARE.
-
---------------------------------------------------------------------------------
-
-3rdparty dependency snappy is statically linked in certain binary
-distributions, like the python wheels. snappy has the following license:
-
-Copyright 2011, Google Inc.
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are
-met:
-
- * Redistributions of source code must retain the above copyright notice,
- this list of conditions and the following disclaimer.
- * Redistributions in binary form must reproduce the above copyright notice,
- this list of conditions and the following disclaimer in the documentation
- and/or other materials provided with the distribution.
- * Neither the name of Google Inc. nor the names of its contributors may be
- used to endorse or promote products derived from this software without
- specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-===
-
-Some of the benchmark data in testdata/ is licensed differently:
-
- - fireworks.jpeg is Copyright 2013 Steinar H. Gunderson, and
- is licensed under the Creative Commons Attribution 3.0 license
- (CC-BY-3.0). See https://creativecommons.org/licenses/by/3.0/
- for more information.
-
- - kppkn.gtb is taken from the Gaviota chess tablebase set, and
- is licensed under the MIT License. See
- https://sites.google.com/site/gaviotachessengine/Home/endgame-tablebases-1
- for more information.
-
- - paper-100k.pdf is an excerpt (bytes 92160 to 194560) from the paper
- “Combinatorial Modeling of Chromatin Features Quantitatively Predicts DNA
- Replication Timing in _Drosophila_” by Federico Comoglio and Renato Paro,
- which is licensed under the CC-BY license. See
- http://www.ploscompbiol.org/static/license for more ifnormation.
-
- - alice29.txt, asyoulik.txt, plrabn12.txt and lcet10.txt are from Project
- Gutenberg. The first three have expired copyrights and are in the public
- domain; the latter does not have expired copyright, but is still in the
- public domain according to the license information
- (http://www.gutenberg.org/ebooks/53).
-
---------------------------------------------------------------------------------
-
-3rdparty dependency gflags is statically linked in certain binary
-distributions, like the python wheels. gflags has the following license:
-
-Copyright (c) 2006, Google Inc.
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are
-met:
-
- * Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
- * Redistributions in binary form must reproduce the above
-copyright notice, this list of conditions and the following disclaimer
-in the documentation and/or other materials provided with the
-distribution.
- * Neither the name of Google Inc. nor the names of its
-contributors may be used to endorse or promote products derived from
-this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
---------------------------------------------------------------------------------
-
-3rdparty dependency glog is statically linked in certain binary
-distributions, like the python wheels. glog has the following license:
-
-Copyright (c) 2008, Google Inc.
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are
-met:
-
- * Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
- * Redistributions in binary form must reproduce the above
-copyright notice, this list of conditions and the following disclaimer
-in the documentation and/or other materials provided with the
-distribution.
- * Neither the name of Google Inc. nor the names of its
-contributors may be used to endorse or promote products derived from
-this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-
-A function gettimeofday in utilities.cc is based on
-
-http://www.google.com/codesearch/p?hl=en#dR3YEbitojA/COPYING&q=GetSystemTimeAsFileTime%20license:bsd
-
-The license of this code is:
-
-Copyright (c) 2003-2008, Jouni Malinen <j@w1.fi> and contributors
-All Rights Reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are
-met:
-
-1. Redistributions of source code must retain the above copyright
- notice, this list of conditions and the following disclaimer.
-
-2. Redistributions in binary form must reproduce the above copyright
- notice, this list of conditions and the following disclaimer in the
- documentation and/or other materials provided with the distribution.
-
-3. Neither the name(s) of the above-listed copyright holder(s) nor the
- names of its contributors may be used to endorse or promote products
- derived from this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
---------------------------------------------------------------------------------
-
-3rdparty dependency re2 is statically linked in certain binary
-distributions, like the python wheels. re2 has the following license:
-
-Copyright (c) 2009 The RE2 Authors. All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are
-met:
-
- * Redistributions of source code must retain the above copyright
- notice, this list of conditions and the following disclaimer.
- * Redistributions in binary form must reproduce the above
- copyright notice, this list of conditions and the following
- disclaimer in the documentation and/or other materials provided
- with the distribution.
- * Neither the name of Google Inc. nor the names of its contributors
- may be used to endorse or promote products derived from this
- software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
---------------------------------------------------------------------------------
-
-3rdparty dependency c-ares is statically linked in certain binary
-distributions, like the python wheels. c-ares has the following license:
-
-# c-ares license
-
-Copyright (c) 2007 - 2018, Daniel Stenberg with many contributors, see AUTHORS
-file.
-
-Copyright 1998 by the Massachusetts Institute of Technology.
-
-Permission to use, copy, modify, and distribute this software and its
-documentation for any purpose and without fee is hereby granted, provided that
-the above copyright notice appear in all copies and that both that copyright
-notice and this permission notice appear in supporting documentation, and that
-the name of M.I.T. not be used in advertising or publicity pertaining to
-distribution of the software without specific, written prior permission.
-M.I.T. makes no representations about the suitability of this software for any
-purpose. It is provided "as is" without express or implied warranty.
-
---------------------------------------------------------------------------------
-
-3rdparty dependency zlib is redistributed as a dynamically linked shared
-library in certain binary distributions, like the python wheels. In the future
-this will likely change to static linkage. zlib has the following license:
-
-zlib.h -- interface of the 'zlib' general purpose compression library
- version 1.2.11, January 15th, 2017
-
- Copyright (C) 1995-2017 Jean-loup Gailly and Mark Adler
-
- This software is provided 'as-is', without any express or implied
- warranty. In no event will the authors be held liable for any damages
- arising from the use of this software.
-
- Permission is granted to anyone to use this software for any purpose,
- including commercial applications, and to alter it and redistribute it
- freely, subject to the following restrictions:
-
- 1. The origin of this software must not be misrepresented; you must not
- claim that you wrote the original software. If you use this software
- in a product, an acknowledgment in the product documentation would be
- appreciated but is not required.
- 2. Altered source versions must be plainly marked as such, and must not be
- misrepresented as being the original software.
- 3. This notice may not be removed or altered from any source distribution.
-
- Jean-loup Gailly Mark Adler
- jloup@gzip.org madler@alumni.caltech.edu
-
---------------------------------------------------------------------------------
-
-3rdparty dependency openssl is redistributed as a dynamically linked shared
-library in certain binary distributions, like the python wheels. openssl
-preceding version 3 has the following license:
-
- LICENSE ISSUES
- ==============
-
- The OpenSSL toolkit stays under a double license, i.e. both the conditions of
- the OpenSSL License and the original SSLeay license apply to the toolkit.
- See below for the actual license texts.
-
- OpenSSL License
- ---------------
-
-/* ====================================================================
- * Copyright (c) 1998-2019 The OpenSSL Project. All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- *
- * 1. Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- *
- * 2. Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in
- * the documentation and/or other materials provided with the
- * distribution.
- *
- * 3. All advertising materials mentioning features or use of this
- * software must display the following acknowledgment:
- * "This product includes software developed by the OpenSSL Project
- * for use in the OpenSSL Toolkit. (http://www.openssl.org/)"
- *
- * 4. The names "OpenSSL Toolkit" and "OpenSSL Project" must not be used to
- * endorse or promote products derived from this software without
- * prior written permission. For written permission, please contact
- * openssl-core@openssl.org.
- *
- * 5. Products derived from this software may not be called "OpenSSL"
- * nor may "OpenSSL" appear in their names without prior written
- * permission of the OpenSSL Project.
- *
- * 6. Redistributions of any form whatsoever must retain the following
- * acknowledgment:
- * "This product includes software developed by the OpenSSL Project
- * for use in the OpenSSL Toolkit (http://www.openssl.org/)"
- *
- * THIS SOFTWARE IS PROVIDED BY THE OpenSSL PROJECT ``AS IS'' AND ANY
- * EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
- * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE OpenSSL PROJECT OR
- * ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
- * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
- * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
- * OF THE POSSIBILITY OF SUCH DAMAGE.
- * ====================================================================
- *
- * This product includes cryptographic software written by Eric Young
- * (eay@cryptsoft.com). This product includes software written by Tim
- * Hudson (tjh@cryptsoft.com).
- *
- */
-
- Original SSLeay License
- -----------------------
-
-/* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com)
- * All rights reserved.
- *
- * This package is an SSL implementation written
- * by Eric Young (eay@cryptsoft.com).
- * The implementation was written so as to conform with Netscapes SSL.
- *
- * This library is free for commercial and non-commercial use as long as
- * the following conditions are aheared to. The following conditions
- * apply to all code found in this distribution, be it the RC4, RSA,
- * lhash, DES, etc., code; not just the SSL code. The SSL documentation
- * included with this distribution is covered by the same copyright terms
- * except that the holder is Tim Hudson (tjh@cryptsoft.com).
- *
- * Copyright remains Eric Young's, and as such any Copyright notices in
- * the code are not to be removed.
- * If this package is used in a product, Eric Young should be given attribution
- * as the author of the parts of the library used.
- * This can be in the form of a textual message at program startup or
- * in documentation (online or textual) provided with the package.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the copyright
- * notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in the
- * documentation and/or other materials provided with the distribution.
- * 3. All advertising materials mentioning features or use of this software
- * must display the following acknowledgement:
- * "This product includes cryptographic software written by
- * Eric Young (eay@cryptsoft.com)"
- * The word 'cryptographic' can be left out if the rouines from the library
- * being used are not cryptographic related :-).
- * 4. If you include any Windows specific code (or a derivative thereof) from
- * the apps directory (application code) you must include an acknowledgement:
- * "This product includes software written by Tim Hudson (tjh@cryptsoft.com)"
- *
- * THIS SOFTWARE IS PROVIDED BY ERIC YOUNG ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- * The licence and distribution terms for any publically available version or
- * derivative of this code cannot be changed. i.e. this code cannot simply be
- * copied and put under another distribution licence
- * [including the GNU Public Licence.]
- */
-
---------------------------------------------------------------------------------
-
-This project includes code from the rtools-backports project.
-
-* ci/scripts/PKGBUILD and ci/scripts/r_windows_build.sh are based on code
- from the rtools-backports project.
-
-Copyright: Copyright (c) 2013 - 2019, Алексей and Jeroen Ooms.
-All rights reserved.
-Homepage: https://github.com/r-windows/rtools-backports
-License: 3-clause BSD
-
---------------------------------------------------------------------------------
-
-Some code from pandas has been adapted for the pyarrow codebase. pandas is
-available under the 3-clause BSD license, which follows:
-
-pandas license
-==============
-
-Copyright (c) 2011-2012, Lambda Foundry, Inc. and PyData Development Team
-All rights reserved.
-
-Copyright (c) 2008-2011 AQR Capital Management, LLC
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are
-met:
-
- * Redistributions of source code must retain the above copyright
- notice, this list of conditions and the following disclaimer.
-
- * Redistributions in binary form must reproduce the above
- copyright notice, this list of conditions and the following
- disclaimer in the documentation and/or other materials provided
- with the distribution.
-
- * Neither the name of the copyright holder nor the names of any
- contributors may be used to endorse or promote products derived
- from this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER AND CONTRIBUTORS
-"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
---------------------------------------------------------------------------------
-
-Some bits from DyND, in particular aspects of the build system, have been
-adapted from libdynd and dynd-python under the terms of the BSD 2-clause
-license
-
-The BSD 2-Clause License
-
- Copyright (C) 2011-12, Dynamic NDArray Developers
- All rights reserved.
-
- Redistribution and use in source and binary forms, with or without
- modification, are permitted provided that the following conditions are
- met:
-
- * Redistributions of source code must retain the above copyright
- notice, this list of conditions and the following disclaimer.
-
- * Redistributions in binary form must reproduce the above
- copyright notice, this list of conditions and the following
- disclaimer in the documentation and/or other materials provided
- with the distribution.
-
- THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-Dynamic NDArray Developers list:
-
- * Mark Wiebe
- * Continuum Analytics
-
---------------------------------------------------------------------------------
-
-Some source code from Ibis (https://github.com/cloudera/ibis) has been adapted
-for PyArrow. Ibis is released under the Apache License, Version 2.0.
-
---------------------------------------------------------------------------------
-
-This project includes code from the autobrew project.
-
-* r/tools/autobrew and dev/tasks/homebrew-formulae/autobrew/apache-arrow.rb
- are based on code from the autobrew project.
-
-Copyright (c) 2019, Jeroen Ooms
-License: MIT
-Homepage: https://github.com/jeroen/autobrew
-
---------------------------------------------------------------------------------
-
-dev/tasks/homebrew-formulae/apache-arrow.rb has the following license:
-
-BSD 2-Clause License
-
-Copyright (c) 2009-present, Homebrew contributors
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are met:
-
-* Redistributions of source code must retain the above copyright notice, this
- list of conditions and the following disclaimer.
-
-* Redistributions in binary form must reproduce the above copyright notice,
- this list of conditions and the following disclaimer in the documentation
- and/or other materials provided with the distribution.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
-FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
-DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
-SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
-CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
-OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-----------------------------------------------------------------------
-
-cpp/src/arrow/vendored/base64.cpp has the following license
-
-ZLIB License
-
-Copyright (C) 2004-2017 René Nyffenegger
-
-This source code is provided 'as-is', without any express or implied
-warranty. In no event will the author be held liable for any damages arising
-from the use of this software.
-
-Permission is granted to anyone to use this software for any purpose, including
-commercial applications, and to alter it and redistribute it freely, subject to
-the following restrictions:
-
-1. The origin of this source code must not be misrepresented; you must not
- claim that you wrote the original source code. If you use this source code
- in a product, an acknowledgment in the product documentation would be
- appreciated but is not required.
-
-2. Altered source versions must be plainly marked as such, and must not be
- misrepresented as being the original source code.
-
-3. This notice may not be removed or altered from any source distribution.
-
-René Nyffenegger rene.nyffenegger@adp-gmbh.ch
-
---------------------------------------------------------------------------------
-
-The file cpp/src/arrow/vendored/optional.hpp has the following license
-
-Boost Software License - Version 1.0 - August 17th, 2003
-
-Permission is hereby granted, free of charge, to any person or organization
-obtaining a copy of the software and accompanying documentation covered by
-this license (the "Software") to use, reproduce, display, distribute,
-execute, and transmit the Software, and to prepare derivative works of the
-Software, and to permit third-parties to whom the Software is furnished to
-do so, all subject to the following:
-
-The copyright notices in the Software and this entire statement, including
-the above license grant, this restriction and the following disclaimer,
-must be included in all copies of the Software, in whole or in part, and
-all derivative works of the Software, unless such copies or derivative
-works are solely in the form of machine-executable object code generated by
-a source language processor.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
-SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
-FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
-ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
-DEALINGS IN THE SOFTWARE.
-
---------------------------------------------------------------------------------
-
-This project includes code from Folly.
-
- * cpp/src/arrow/vendored/ProducerConsumerQueue.h
-
-is based on Folly's
-
- * folly/Portability.h
- * folly/lang/Align.h
- * folly/ProducerConsumerQueue.h
-
-Copyright: Copyright (c) Facebook, Inc. and its affiliates.
-Home page: https://github.com/facebook/folly
-License: http://www.apache.org/licenses/LICENSE-2.0
-
---------------------------------------------------------------------------------
-
-The file cpp/src/arrow/vendored/musl/strptime.c has the following license
-
-Copyright © 2005-2020 Rich Felker, et al.
-
-Permission is hereby granted, free of charge, to any person obtaining
-a copy of this software and associated documentation files (the
-"Software"), to deal in the Software without restriction, including
-without limitation the rights to use, copy, modify, merge, publish,
-distribute, sublicense, and/or sell copies of the Software, and to
-permit persons to whom the Software is furnished to do so, subject to
-the following conditions:
-
-The above copyright notice and this permission notice shall be
-included in all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
-IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
-CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
-TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
-SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
-
---------------------------------------------------------------------------------
-
-The file cpp/cmake_modules/BuildUtils.cmake contains code from
-
-https://gist.github.com/cristianadam/ef920342939a89fae3e8a85ca9459b49
-
-which is made available under the MIT license
-
-Copyright (c) 2019 Cristian Adam
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.
-
---------------------------------------------------------------------------------
-
-The files in cpp/src/arrow/vendored/portable-snippets/ contain code from
-
-https://github.com/nemequ/portable-snippets
-
-and have the following copyright notice:
-
-Each source file contains a preamble explaining the license situation
-for that file, which takes priority over this file. With the
-exception of some code pulled in from other repositories (such as
-µnit, an MIT-licensed project which is used for testing), the code is
-public domain, released using the CC0 1.0 Universal dedication (*).
-
-(*) https://creativecommons.org/publicdomain/zero/1.0/legalcode
-
---------------------------------------------------------------------------------
-
-The files in cpp/src/arrow/vendored/fast_float/ contain code from
-
-https://github.com/lemire/fast_float
-
-which is made available under the Apache License 2.0.
-
---------------------------------------------------------------------------------
-
-The file python/pyarrow/vendored/version.py contains code from
-
-https://github.com/pypa/packaging/
-
-which is made available under both the Apache license v2.0 and the
-BSD 2-clause license.
diff --git a/README.md b/README.md
deleted file mode 100644
index 5d6a58a..0000000
--- a/README.md
+++ /dev/null
@@ -1,24 +0,0 @@
-<!---
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
--->
-
-# Experimental safe parquet
-
-This repository contains experimental code to read from parquet safely.
-
-This is _not_ officially supported code from Apache Arrow.
diff --git a/dev/.gitignore b/dev/.gitignore
deleted file mode 100644
index c03a7c7..0000000
--- a/dev/.gitignore
+++ /dev/null
@@ -1,22 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-# Python virtual environments for dev tools
-.venv*/
-
-__pycache__
-*.egg-info
diff --git a/dev/README.md b/dev/README.md
deleted file mode 100644
index b4ea02b..0000000
--- a/dev/README.md
+++ /dev/null
@@ -1,57 +0,0 @@
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one
- ~ or more contributor license agreements. See the NOTICE file
- ~ distributed with this work for additional information
- ~ regarding copyright ownership. The ASF licenses this file
- ~ to you under the Apache License, Version 2.0 (the
- ~ "License"); you may not use this file except in compliance
- ~ with the License. You may obtain a copy of the License at
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing,
- ~ software distributed under the License is distributed on an
- ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- ~ KIND, either express or implied. See the License for the
- ~ specific language governing permissions and limitations
- ~ under the License.
- -->
-
-# Arrow Developer Scripts
-
-This directory contains scripts useful to developers when packaging,
-testing, or committing to Arrow.
-
-## Verifying Release Candidates
-
-We have provided a script to assist with verifying release candidates:
-
-```shell
-bash dev/release/verify-release-candidate.sh 0.7.0 0
-```
-
-Currently this only works on Linux (patches to expand to macOS welcome!). Read
-the script for information about system dependencies.
-
-On Windows, we have a script that verifies C++ and Python (requires Visual
-Studio 2015):
-
-```
-dev/release/verify-release-candidate.bat apache-arrow-0.7.0.tar.gz
-```
-
-### Verifying the JavaScript release
-
-For JavaScript-specific releases, use a different verification script:
-
-```shell
-bash dev/release/js-verify-release-candidate.sh 0.7.0 0
-```
-
-# Integration testing
-
-Build the following base image used by multiple tests:
-
-```shell
-docker build -t arrow_integration_xenial_base -f docker_common/Dockerfile.xenial.base .
-```
diff --git a/dev/release/README.md b/dev/release/README.md
deleted file mode 100644
index 5b31083..0000000
--- a/dev/release/README.md
+++ /dev/null
@@ -1,266 +0,0 @@
-<!---
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
--->
-
-# Release Process
-
-## Branching
-
-We would maintain two branches: `active_release` and `master`.
-
-- All new PRs are created and merged against `master`
-- All versions are created from the `active_release` branch
-- Once merged to master, changes are "cherry-picked" (via a hopefully soon to be automated process), to the `active_release` branch based on the judgement of the original PR author and maintainers.
-
-- We do not merge breaking api changes, as defined in [Rust RFC 1105](https://github.com/rust-lang/rfcs/blob/master/text/1105-api-evolution.md) to the `active_release`
-
-Please see the [original proposal](https://docs.google.com/document/d/1tMQ67iu8XyGGZuj--h9WQYB9inCk6c2sL_4xMTwENGc/edit?ts=60961758) document the rational of this change.
-
-## Release Branching
-
-We aim to release every other week from the `active_release` branch.
-
-Every other Monday, a maintainer proposes a minor (e.g. `4.1.0` to `4.2.0`) or patch (e.g `4.1.0` to `4.1.1`) release, depending on changes to the `active_release` in the previous 2 weeks, following the process beloe.
-
-If this release is approved by at least three PMC members, a new version from that tarball is released to crates.io later in the week.
-
-Apache Arrow in general does synchronized major releases every three months. The Rust implementation aims to do its major releases in the same time frame.
-
-# Release Mechanics
-
-This directory contains the scripts used to manage an Apache Arrow Release.
-
-# Process Overview
-
-As part of the Apache governance model, official releases consist of
-signed source tarballs approved by the PMC.
-
-We then use the code in the approved source tarball to release to
-crates.io, the Rust ecosystem's package manager.
-
-## Branching
-
-# Release Preparation
-
-# Change Log
-
-We create a `CHANGELOG.md` so our users know what has been changed between releases.
-
-The CHANGELOG is created automatically using
-[change_log.sh](https://github.com/apache/arrow-rs/blob/master/change_log.sh)
-
-This script creates a changelog using github issues and the
-labels associated with them.
-
-# Mechanics of creating a release
-
-## Prepare the release branch and tags
-
-First, ensure that `active_release` contains the content of the desired release. For minor and patch releases, no additional steps are needed.
-
-To prepare for _a major release_, change `active release` to point at the latest `master` with commands such as:
-
-```
-git checkout active_release
-git fetch apache
-git reset --hard apache/master
-git push -f
-```
-
-### Update CHANGELOG.md + Version
-
-Now prepare a PR to update `CHANGELOG.md` and versions on `active_release` branch to reflect the planned release.
-
-See [#298](https://github.com/apache/arrow-rs/pull/298) for an example.
-
-Here are the commands used to prepare the 4.1.0 release:
-
-```bash
-git checkout active_release
-git pull
-git checkout -b make-release
-
-# manully edit ./dev/release/update_change_log.sh to reflect the release version
-# create the changelog
-CHANGELOG_GITHUB_TOKEN=<TOKEN> ./dev/release/update_change_log.sh
-# review change log / edit issues and labels if needed, rerun
-git commit -a -m 'Create changelog'
-
-# update versions
-sed -i '' -e 's/5.0.0-SNAPSHOT/4.1.0/g' `find . -name 'Cargo.toml'`
-git commit -a -m 'Update version'
-```
-
-Note that when reviewing the change log, rather than editing the
-`CHANGELOG.md`, it is preferred to update the issues and their labels
-(e.g. add `invalid` label to exclude them from release notes)
-
-## Prepare release candidate tarball
-
-(Note you need to be a committer to run these scripts as they upload to the apache svn distribution servers)
-
-### Create git tag for the release:
-
-While the official release artifact is a signed tarball, we also tag the commit it was created for convenience and code archaeology.
-
-Using a string such as `4.0.1` as the `<version>`, create and push the tag thusly:
-
-```shell
-git fetch apache
-git tag <version> apache/active_release
-# push tag to apache
-git push apache <version>
-```
-
-### Pick an Release Candidate (RC) number
-
-Pick numbers in sequential order, with `1` for `rc1`, `2` for `rc2`, etc.
-
-### Create, sign, and upload tarball
-
-Run the `create-tarball.sh` with the `<version>` tag and `<rc>` and you found in previous steps:
-
-```shell
-./dev/release/create-tarball.sh 4.1.0 2
-```
-
-This script
-
-1. creates and uploads a release candidate tarball to the [arrow
- dev](https://dist.apache.org/repos/dist/dev/arrow) location on the
- apache distribution svn server
-
-2. provide you an email template to
- send to dev@arrow.apache.org for release voting.
-
-### Vote on Release Candidate tarball
-
-Send the email output from the script to dev@arrow.apache.org. The email should look like
-
-```
-To: dev@arrow.apache.org
-Subject: [VOTE][RUST] Release Apache Arrow
-
-Hi,
-
-I would like to propose a release of Apache Arrow Rust
-Implementation, version 4.1.0.
-
-This release candidate is based on commit: a5dd428f57e62db20a945e8b1895de91405958c4 [1]
-
-The proposed release tarball and signatures are hosted at [2].
-The changelog is located at [3].
-
-Please download, verify checksums and signatures, run the unit tests,
-and vote on the release.
-
-The vote will be open for at least 72 hours.
-
-[ ] +1 Release this as Apache Arrow Rust
-[ ] +0
-[ ] -1 Do not release this as Apache Arrow Rust because...
-
-[1]: https://github.com/apache/arrow-rs/tree/a5dd428f57e62db20a945e8b1895de91405958c4
-[2]: https://dist.apache.org/repos/dist/dev/arrow/apache-arrow-rs-4.1.0
-[3]: https://github.com/apache/arrow-rs/blob/a5dd428f57e62db20a945e8b1895de91405958c4/CHANGELOG.md
-```
-
-For the release to become "official" it needs at least three PMC members to vote +1 on it.
-
-#### Verifying Release Candidates
-
-There is a script in this repository which can be used to help `dev/release/verify-release-candidate.sh` assist the verification process. Run it like:
-
-```
-./dev/release/verify-release-candidate.sh 4.1.0 2
-```
-
-#### If the release is not approved
-
-If the release is not approved, fix whatever the problem is and try again with the next RC number
-
-### If the release is approved,
-
-Move tarball to the release location in SVN, e.g. https://dist.apache.org/repos/dist/release/arrow/arrow-4.1.0/, using the `release-tarball.sh` script:
-
-```shell
-./dev/release/release-tarball.sh 4.1.0 2
-```
-
-### Publish on Crates.io
-
-Only approved releases of the tarball should be published to
-crates.io, in order to conform to Apache Software Foundation
-governance standards.
-
-An Arrow committer can publish this crate after an official project release has
-been made to crates.io using the following instructions.
-
-Follow [these
-instructions](https://doc.rust-lang.org/cargo/reference/publishing.html) to
-create an account and login to crates.io before asking to be added as an owner
-of the [arrow crate](https://crates.io/crates/arrow).
-
-Download and unpack the official release tarball
-
-Verify that the Cargo.toml in the tarball contains the correct version
-(e.g. `version = "0.11.0"`) and then publish the crate with the
-following commands
-
-```shell
-(cd arrow && cargo publish)
-(cd arrow_flight && cargo publish)
-(cd parquet && cargo publish)
-(cd parquet_derive && cargo publish)
-```
-
-# Backporting
-
-As of now, the plan for backporting to `active_release` is to do so semi-manually.
-
-Step 1: Pick the commit to cherry-pick
-
-Step 2: Create cherry-pick PR to active_release
-
-Step 3a: If CI passes, merge cherry-pick PR
-
-Step 3b: If CI doesn't pass or some other changes are needed, the PR should be reviewed / approved as normal prior to merge
-
-For example, to backport `b2de5446cc1e45a0559fb39039d0545df1ac0d26` to active_release use the folliwing
-
-```shell
-git clone git@github.com:apache/arrow-rs.git /tmp/arrow-rs
-
-ARROW_GITHUB_API_TOKEN=$ARROW_GITHUB_API_TOKEN CHECKOUT_ROOT=/tmp/arrow-rs CHERRY_PICK_SHA=b2de5446cc1e45a0559fb39039d0545df1ac0d26 python3 dev/release/cherry-pick-pr.py
-```
-
-## Tags
-
-There are two tags that help keep track of backporting:
-
-1. [`cherry-picked`](https://github.com/apache/arrow-rs/labels/cherry-picked) for PRs that have been cherry-picked/backported to `active_release`
-2. [`release-cherry-pick`](https://github.com/apache/arrow-rs/labels/release-cherry-pick) for the PRs that are the cherry pick
-
-You can find candidates to cherry pick using [this filter](https://github.com/apache/arrow-rs/pulls?q=is%3Apr+is%3Aclosed+-label%3Arelease-cherry-pick+-label%3Acherry-picked)
-
-## Rationale for creating PRs:
-
-1. PRs are a natural place to run the CI tests to make sure there are no logical conflicts
-2. PRs offer a place for the original author / committers to comment and say it should/should not be backported.
-3. PRs offer a way to make cleanups / fixups and approve (if needed) for non cherry pick PRs
-4. There is an additional control / review when the candidate release is created
diff --git a/dev/release/check-rat-report.py b/dev/release/check-rat-report.py
deleted file mode 100644
index e30d72b..0000000
--- a/dev/release/check-rat-report.py
+++ /dev/null
@@ -1,59 +0,0 @@
-#!/usr/bin/python
-##############################################################################
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-##############################################################################
-import fnmatch
-import re
-import sys
-import xml.etree.ElementTree as ET
-
-if len(sys.argv) != 3:
- sys.stderr.write("Usage: %s exclude_globs.lst rat_report.xml\n" %
- sys.argv[0])
- sys.exit(1)
-
-exclude_globs_filename = sys.argv[1]
-xml_filename = sys.argv[2]
-
-globs = [line.strip() for line in open(exclude_globs_filename, "r")]
-
-tree = ET.parse(xml_filename)
-root = tree.getroot()
-resources = root.findall('resource')
-
-all_ok = True
-for r in resources:
- approvals = r.findall('license-approval')
- if not approvals or approvals[0].attrib['name'] == 'true':
- continue
- clean_name = re.sub('^[^/]+/', '', r.attrib['name'])
- excluded = False
- for g in globs:
- if fnmatch.fnmatch(clean_name, g):
- excluded = True
- break
- if not excluded:
- sys.stdout.write("NOT APPROVED: %s (%s): %s\n" % (
- clean_name, r.attrib['name'], approvals[0].attrib['name']))
- all_ok = False
-
-if not all_ok:
- sys.exit(1)
-
-print('OK')
-sys.exit(0)
diff --git a/dev/release/cherry-pick-pr.py b/dev/release/cherry-pick-pr.py
deleted file mode 100755
index 2886a0d..0000000
--- a/dev/release/cherry-pick-pr.py
+++ /dev/null
@@ -1,154 +0,0 @@
-#!/usr/bin/python3
-##############################################################################
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-##############################################################################
-
-# This script is designed to create a cherry pick PR to a target branch
-#
-# Usage: python3 cherry_pick_pr.py
-#
-# To test locally:
-#
-# git clone git@github.com:apache/arrow-rs.git /tmp/arrow-rs
-#
-# pip3 install PyGithub
-# ARROW_GITHUB_API_TOKEN=<..>
-# CHECKOUT_ROOT=<path>
-# CHERRY_PICK_SHA=<sha> python3 cherry-pick-pr.py
-#
-import os
-import sys
-import six
-import subprocess
-
-from pathlib import Path
-
-TARGET_BRANCH = 'active_release'
-TARGET_REPO = 'apache/arrow-rs'
-
-p = Path(__file__)
-
-# Use github workspace if specified
-repo_root = os.environ.get("CHECKOUT_ROOT")
-if repo_root is None:
- print("arrow-rs checkout must be supplied in CHECKOUT_ROOT environment")
- sys.exit(1)
-
-print("Using checkout in {}".format(repo_root))
-
-token = os.environ.get('ARROW_GITHUB_API_TOKEN', None)
-if token is None:
- print("GITHUB token must be supplied in ARROW_GITHUB_API_TOKEN environmet")
- sys.exit(1)
-
-new_sha = os.environ.get('CHERRY_PICK_SHA', None)
-if new_sha is None:
- print("SHA to cherry pick must be supplied in CHERRY_PICK_SHA environment")
- sys.exit(1)
-
-
-# from merge_pr.py from arrow repo
-def run_cmd(cmd):
- if isinstance(cmd, six.string_types):
- cmd = cmd.split(' ')
- try:
- output = subprocess.check_output(cmd)
- except subprocess.CalledProcessError as e:
- # this avoids hiding the stdout / stderr of failed processes
- print('Command failed: %s' % cmd)
- print('With output:')
- print('--------------')
- print(e.output)
- print('--------------')
- raise e
-
- if isinstance(output, six.binary_type):
- output = output.decode('utf-8')
-
- return output
-
-
-os.chdir(repo_root)
-new_sha_short = run_cmd("git rev-parse --short {}".format(new_sha)).strip()
-new_branch = 'cherry_pick_{}'.format(new_sha_short)
-
-
-def make_cherry_pick():
- if os.environ.get('GITHUB_SHA', None) is not None:
- print("Running on github runner, setting email/username")
- run_cmd(['git', 'config', 'user.email', 'dev@arrow.apache.com'])
- run_cmd(['git', 'config', 'user.name', 'Arrow-RS Automation'])
-
- #
- # Create a new branch from active_release
- # and cherry pick to there.
- #
-
- print("Creating cherry pick from {} to {}".format(
- new_sha_short, new_branch
- ))
-
- # The following tortured dance is required due to how the github
- # actions/checkout works (it doesn't pull other branches and pulls
- # only one commit back)
-
- # pull 10 commits back so we can get the proper cherry pick
- # (probably only need 2 but 10 must be better, right?)
- run_cmd(['git', 'fetch', '--depth', '10', 'origin', 'master'])
- run_cmd(['git', 'fetch', 'origin', 'active_release'])
- run_cmd(['git', 'checkout', '-b', new_branch])
- run_cmd(['git', 'reset', '--hard', 'origin/active_release'])
- run_cmd(['git', 'cherry-pick', new_sha])
- run_cmd(['git', 'push', '-u', 'origin', new_branch])
-
-
-
-def make_cherry_pick_pr():
- from github import Github
- g = Github(token)
- repo = g.get_repo(TARGET_REPO)
-
- release_cherry_pick_label = repo.get_label('release-cherry-pick')
- cherry_picked_label = repo.get_label('cherry-picked')
-
- # Default titles
- new_title = 'Cherry pick {} to active_release'.format(new_sha)
- new_commit_message = 'Automatic cherry-pick of {}\n'.format(new_sha)
-
- # try and get info from github api
- commit = repo.get_commit(new_sha)
- for orig_pull in commit.get_pulls():
- new_commit_message += '* Originally appeared in {}: {}\n'.format(
- orig_pull.html_url, orig_pull.title)
- new_title = 'Cherry pick {} to active_release'.format(orig_pull.title)
- orig_pull.add_to_labels(cherry_picked_label)
-
- pr = repo.create_pull(title=new_title,
- body=new_commit_message,
- base='refs/heads/active_release',
- head='refs/heads/{}'.format(new_branch),
- maintainer_can_modify=True,
- )
-
- pr.add_to_labels(release_cherry_pick_label)
-
- print('Created PR {}'.format(pr.html_url))
-
-
-make_cherry_pick()
-make_cherry_pick_pr()
diff --git a/dev/release/create-tarball.sh b/dev/release/create-tarball.sh
deleted file mode 100755
index 2311519..0000000
--- a/dev/release/create-tarball.sh
+++ /dev/null
@@ -1,123 +0,0 @@
-#!/bin/bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-
-# This script creates a signed tarball in
-# dev/dist/apache-arrow-rs-<version>-<sha>.tar.gz and uploads it to
-# the "dev" area of the dist.apache.arrow repository and prepares an
-# email for sending to the dev@arrow.apache.org list for a formal
-# vote.
-#
-# See release/README.md for full release instructions
-#
-# Requirements:
-#
-# 1. gpg setup for signing and have uploaded your public
-# signature to https://pgp.mit.edu/
-#
-# 2. Logged into the apache svn server with the appropriate
-# credentials
-#
-#
-# Based in part on 02-source.sh from apache/arrow
-#
-
-set -e
-
-SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-SOURCE_TOP_DIR="$(cd "${SOURCE_DIR}/../../" && pwd)"
-
-if [ "$#" -ne 2 ]; then
- echo "Usage: $0 <tag> <rc>"
- echo "ex. $0 4.1.0 2"
- exit
-fi
-
-tag=$1
-rc=$2
-
-release_hash=$(cd "${SOURCE_TOP_DIR}" && git rev-list --max-count=1 ${tag})
-
-release=apache-arrow-rs-${tag}
-distdir=${SOURCE_TOP_DIR}/dev/dist/${release}-rc${rc}
-tarname=${release}.tar.gz
-tarball=${distdir}/${tarname}
-url="https://dist.apache.org/repos/dist/dev/arrow/${release}-rc${rc}"
-
-echo "Attempting to create ${tarball} from tag ${tag}"
-
-
-if [ -z "$release_hash" ]; then
- echo "Cannot continue: unknown git tag: $tag"
-fi
-
-echo "Draft email for dev@arrow.apache.org mailing list"
-echo ""
-echo "---------------------------------------------------------"
-cat <<MAIL
-To: dev@arrow.apache.org
-Subject: [VOTE][RUST] Release Apache Arrow Rust ${tag} RC${rc}
-
-Hi,
-
-I would like to propose a release of Apache Arrow Rust Implementation, version ${tag}.
-
-This release candidate is based on commit: ${release_hash} [1]
-
-The proposed release tarball and signatures are hosted at [2].
-
-The changelog is located at [3].
-
-Please download, verify checksums and signatures, run the unit tests,
-and vote on the release.
-
-The vote will be open for at least 72 hours.
-
-[ ] +1 Release this as Apache Arrow Rust ${version}
-[ ] +0
-[ ] -1 Do not release this as Apache Arrow Rust ${version} because...
-
-[1]: https://github.com/apache/arrow-rs/tree/${release_hash}
-[2]: ${url}
-[3]: https://github.com/apache/arrow-rs/blob/${release_hash}/CHANGELOG.md
-MAIL
-echo "---------------------------------------------------------"
-
-
-# create <tarball> containing the files in git at $release_hash
-# the files in the tarball are prefixed with {tag} (e.g. 4.0.1)
-mkdir -p ${distdir}
-(cd "${SOURCE_TOP_DIR}" && git archive ${release_hash} --prefix ${release}/ | gzip > ${tarball})
-
-echo "Running rat license checker on ${tarball}"
-${SOURCE_DIR}/run-rat.sh ${tarball}
-
-echo "Signing tarball and creating checksums"
-gpg --armor --output ${tarball}.asc --detach-sig ${tarball}
-# create signing with relative path of tarball
-# so that they can be verified with a command such as
-# shasum --check apache-arrow-rs-4.1.0-rc2.tar.gz.sha512
-(cd ${distdir} && shasum -a 256 ${tarname}) > ${tarball}.sha256
-(cd ${distdir} && shasum -a 512 ${tarname}) > ${tarball}.sha512
-
-echo "Uploading to apache dist/dev to ${url}"
-svn co --depth=empty https://dist.apache.org/repos/dist/dev/arrow ${SOURCE_TOP_DIR}/dev/dist
-svn add ${distdir}
-svn ci -m "Apache Arrow Rust ${tag} ${rc}" ${distdir}
diff --git a/dev/release/rat_exclude_files.txt b/dev/release/rat_exclude_files.txt
deleted file mode 100644
index d2b5aad..0000000
--- a/dev/release/rat_exclude_files.txt
+++ /dev/null
@@ -1,27 +0,0 @@
-*.npmrc
-*.gitignore
-.gitmodules
-*.csv
-*.json
-*.snap
-.github/ISSUE_TEMPLATE/*.md
-.github/pull_request_template.md
-CHANGELOG.md
-pax_global_header
-MANIFEST.in
-requirements.txt
-*.html
-*.sgml
-*.css
-*.png
-*.ico
-*.svg
-*.devhelp2
-*.scss
-.gitattributes
-arrow/test/data/*.csv
-rust-toolchain
-arrow-flight/src/arrow.flight.protocol.rs
-ballista/rust/benchmarks/tpch/queries/q*.sql
-ballista/rust/scheduler/testdata/*
-ballista/ui/scheduler/yarn.lock
diff --git a/dev/release/release-tarball.sh b/dev/release/release-tarball.sh
deleted file mode 100755
index 9612921..0000000
--- a/dev/release/release-tarball.sh
+++ /dev/null
@@ -1,72 +0,0 @@
-#!/bin/bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-# This script copies a tarball from the "dev" area of the
-# dist.apache.arrow repository to the "release" area
-#
-# This script should only be run after the release has been approved
-# by the arrow PMC committee.
-#
-# See release/README.md for full release instructions
-#
-# Based in part on post-01-upload.sh from apache/arrow
-
-
-set -e
-set -u
-
-if [ "$#" -ne 2 ]; then
- echo "Usage: $0 <version> <rc-num>"
- echo "ex. $0 4.1.0 2"
- exit
-fi
-
-version=$1
-rc=$2
-
-tmp_dir=tmp-apache-arrow-dist
-
-echo "Recreate temporary directory: ${tmp_dir}"
-rm -rf ${tmp_dir}
-mkdir -p ${tmp_dir}
-
-echo "Clone dev dist repository"
-svn \
- co \
- https://dist.apache.org/repos/dist/dev/arrow/apache-arrow-rs-${version}-rc${rc} \
- ${tmp_dir}/dev
-
-echo "Clone release dist repository"
-svn co https://dist.apache.org/repos/dist/release/arrow ${tmp_dir}/release
-
-echo "Copy ${version}-rc${rc} to release working copy"
-release_version=arrow-rs-${version}
-mkdir -p ${tmp_dir}/release/${release_version}
-cp -r ${tmp_dir}/dev/* ${tmp_dir}/release/${release_version}/
-svn add ${tmp_dir}/release/${release_version}
-
-echo "Commit release"
-svn ci -m "Apache Arrow Rust ${version}" ${tmp_dir}/release
-
-echo "Clean up"
-rm -rf ${tmp_dir}
-
-echo "Success! The release is available here:"
-echo " https://dist.apache.org/repos/dist/release/arrow/${release_version}"
diff --git a/dev/release/run-rat.sh b/dev/release/run-rat.sh
deleted file mode 100755
index 94fa55f..0000000
--- a/dev/release/run-rat.sh
+++ /dev/null
@@ -1,43 +0,0 @@
-#!/bin/bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-RAT_VERSION=0.13
-
-# download apache rat
-if [ ! -f apache-rat-${RAT_VERSION}.jar ]; then
- curl -s https://repo1.maven.org/maven2/org/apache/rat/apache-rat/${RAT_VERSION}/apache-rat-${RAT_VERSION}.jar > apache-rat-${RAT_VERSION}.jar
-fi
-
-RAT="java -jar apache-rat-${RAT_VERSION}.jar -x "
-
-RELEASE_DIR=$(cd "$(dirname "$BASH_SOURCE")"; pwd)
-
-# generate the rat report
-$RAT $1 > rat.txt
-python $RELEASE_DIR/check-rat-report.py $RELEASE_DIR/rat_exclude_files.txt rat.txt > filtered_rat.txt
-cat filtered_rat.txt
-UNAPPROVED=`cat filtered_rat.txt | grep "NOT APPROVED" | wc -l`
-
-if [ "0" -eq "${UNAPPROVED}" ]; then
- echo "No unapproved licenses"
-else
- echo "${UNAPPROVED} unapproved licences. Check rat report: rat.txt"
- exit 1
-fi
diff --git a/dev/release/update_change_log.sh b/dev/release/update_change_log.sh
deleted file mode 100755
index a11f966..0000000
--- a/dev/release/update_change_log.sh
+++ /dev/null
@@ -1,39 +0,0 @@
-#!/bin/bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-# invokes the changelog generator with the config located in
-# arrow-rs/.github_changelog_generator
-#
-# Usage:
-# CHANGELOG_GITHUB_TOKEN=<TOKEN> ./update_change_log.sh
-
-set -e
-
-SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-SOURCE_TOP_DIR="$(cd "${SOURCE_DIR}/../../" && pwd)"
-
-pushd ${SOURCE_TOP_DIR}
-docker run -it --rm -e CHANGELOG_GITHUB_TOKEN=$CHANGELOG_GITHUB_TOKEN -v "$(pwd)":/usr/local/src/your-app githubchangeloggenerator/github-changelog-generator \
- --user apache \
- --project arrow-rs \
- --since-commit 2021-04-20 \
- --future-release 4.1.0
-
-sed -i "s/\\\n/\n\n/" CHANGELOG.md
diff --git a/dev/release/verify-release-candidate.sh b/dev/release/verify-release-candidate.sh
deleted file mode 100755
index 6c265fb..0000000
--- a/dev/release/verify-release-candidate.sh
+++ /dev/null
@@ -1,153 +0,0 @@
-#!/bin/bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-case $# in
- 2) VERSION="$1"
- RC_NUMBER="$2"
- ;;
- *) echo "Usage: $0 X.Y.Z RC_NUMBER"
- exit 1
- ;;
-esac
-
-set -e
-set -x
-set -o pipefail
-
-SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")" && pwd)"
-ARROW_DIR="$(dirname $(dirname ${SOURCE_DIR}))"
-ARROW_DIST_URL='https://dist.apache.org/repos/dist/dev/arrow'
-
-download_dist_file() {
- curl \
- --silent \
- --show-error \
- --fail \
- --location \
- --remote-name $ARROW_DIST_URL/$1
-}
-
-download_rc_file() {
- download_dist_file apache-arrow-rs-${VERSION}-rc${RC_NUMBER}/$1
-}
-
-import_gpg_keys() {
- download_dist_file KEYS
- gpg --import KEYS
-}
-
-fetch_archive() {
- local dist_name=$1
- download_rc_file ${dist_name}.tar.gz
- download_rc_file ${dist_name}.tar.gz.asc
- download_rc_file ${dist_name}.tar.gz.sha256
- download_rc_file ${dist_name}.tar.gz.sha512
- gpg --verify ${dist_name}.tar.gz.asc ${dist_name}.tar.gz
- shasum -a 256 -c ${dist_name}.tar.gz.sha256
- shasum -a 512 -c ${dist_name}.tar.gz.sha512
-}
-
-verify_dir_artifact_signatures() {
- # verify the signature and the checksums of each artifact
- find $1 -name '*.asc' | while read sigfile; do
- artifact=${sigfile/.asc/}
- gpg --verify $sigfile $artifact || exit 1
-
- # go into the directory because the checksum files contain only the
- # basename of the artifact
- pushd $(dirname $artifact)
- base_artifact=$(basename $artifact)
- if [ -f $base_artifact.sha256 ]; then
- shasum -a 256 -c $base_artifact.sha256 || exit 1
- fi
- shasum -a 512 -c $base_artifact.sha512 || exit 1
- popd
- done
-}
-
-setup_tempdir() {
- cleanup() {
- if [ "${TEST_SUCCESS}" = "yes" ]; then
- rm -fr "${ARROW_TMPDIR}"
- else
- echo "Failed to verify release candidate. See ${ARROW_TMPDIR} for details."
- fi
- }
-
- if [ -z "${ARROW_TMPDIR}" ]; then
- # clean up automatically if ARROW_TMPDIR is not defined
- ARROW_TMPDIR=$(mktemp -d -t "$1.XXXXX")
- trap cleanup EXIT
- else
- # don't clean up automatically
- mkdir -p "${ARROW_TMPDIR}"
- fi
-}
-
-test_source_distribution() {
- # install rust toolchain in a similar fashion like test-miniconda
- export RUSTUP_HOME=$PWD/test-rustup
- export CARGO_HOME=$PWD/test-rustup
-
- curl https://sh.rustup.rs -sSf | sh -s -- -y --no-modify-path
-
- export PATH=$RUSTUP_HOME/bin:$PATH
- source $RUSTUP_HOME/env
-
- # build and test rust
-
- # raises on any formatting errors
- rustup component add rustfmt --toolchain stable
- cargo fmt --all -- --check
-
- # Clone testing repositories if not cloned already
- git clone https://github.com/apache/arrow-testing.git arrow-testing-data
- git clone https://github.com/apache/parquet-testing.git parquet-testing-data
- export ARROW_TEST_DATA=$PWD/arrow-testing-data/data
- export PARQUET_TEST_DATA=$PWD/parquet-testing-data/data
-
- # use local modules because we don't publish modules to crates.io yet
- sed \
- -i.bak \
- -E \
- -e 's/^parquet = "([^"]*)"/parquet = { version = "\1", path = "..\/parquet" }/g' \
- */Cargo.toml
-
- cargo build
- cargo test
-}
-
-TEST_SUCCESS=no
-
-setup_tempdir "arrow-${VERSION}"
-echo "Working in sandbox ${ARROW_TMPDIR}"
-cd ${ARROW_TMPDIR}
-
-dist_name="apache-arrow-rs-${VERSION}"
-import_gpg_keys
-fetch_archive ${dist_name}
-tar xf ${dist_name}.tar.gz
-pushd ${dist_name}
-test_source_distribution
-popd
-
-TEST_SUCCESS=yes
-echo 'Release candidate looks good!'
-exit 0
diff --git a/header b/header
deleted file mode 100644
index 70665d1..0000000
--- a/header
+++ /dev/null
@@ -1,16 +0,0 @@
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
-http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
diff --git a/parquet-testing b/parquet-testing
deleted file mode 160000
index 8e7badc..0000000
--- a/parquet-testing
+++ /dev/null
@@ -1 +0,0 @@
-Subproject commit 8e7badc6a3817a02e06d17b5d8ab6b6dc356e890
diff --git a/parquet/Cargo.toml b/parquet/Cargo.toml
deleted file mode 100644
index cfe4e41..0000000
--- a/parquet/Cargo.toml
+++ /dev/null
@@ -1,82 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-[package]
-name = "parquet"
-version = "5.0.0-SNAPSHOT"
-license = "Apache-2.0"
-description = "Apache Parquet implementation in Rust"
-homepage = "https://github.com/apache/arrow-rs"
-repository = "https://github.com/apache/arrow-rs"
-authors = ["Apache Arrow <dev@arrow.apache.org>"]
-keywords = [ "arrow", "parquet", "hadoop" ]
-readme = "README.md"
-build = "build.rs"
-edition = "2018"
-
-[dependencies]
-# update note: pin `parquet-format` to specific version until it does not break at minor
-# version, see ARROW-11187.
-parquet-format = "~2.6.1"
-byteorder = "1"
-thrift = "0.13"
-snap = { version = "1.0", optional = true }
-brotli = { version = "3.3", optional = true }
-flate2 = { version = "1.0", optional = true }
-lz4 = { version = "1.23", optional = true }
-zstd = { version = "0.8", optional = true }
-chrono = "0.4"
-num-bigint = "0.4"
-arrow = { version = "4.3.0", optional = true }
-base64 = { version = "0.13", optional = true }
-clap = { version = "2.33.3", optional = true }
-serde_json = { version = "1.0", features = ["preserve_order"], optional = true }
-rand = "0.8"
-
-[dev-dependencies]
-criterion = "0.3"
-rand = "0.8"
-snap = "1.0"
-brotli = "3.3"
-flate2 = "1.0"
-lz4 = "1.23"
-zstd = "0.8"
-serde_json = { version = "1.0", features = ["preserve_order"] }
-
-[features]
-default = ["arrow", "snap", "brotli", "flate2", "lz4", "zstd", "base64"]
-cli = ["serde_json", "base64", "clap"]
-
-[[ bin ]]
-name = "parquet-read"
-required-features = ["cli"]
-
-[[ bin ]]
-name = "parquet-schema"
-required-features = ["cli"]
-
-[[ bin ]]
-name = "parquet-rowcount"
-required-features = ["cli"]
-
-[[bench]]
-name = "arrow_writer"
-harness = false
-
-[[bench]]
-name = "arrow_array_reader"
-harness = false
\ No newline at end of file
diff --git a/parquet/README.md b/parquet/README.md
deleted file mode 100644
index 338dba7..0000000
--- a/parquet/README.md
+++ /dev/null
@@ -1,143 +0,0 @@
-<!---
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
--->
-
-# An Apache Parquet implementation in Rust
-
-[](https://crates.io/crates/parquet)
-
-## Usage
-
-Add this to your Cargo.toml:
-
-```toml
-[dependencies]
-parquet = "^4"
-```
-
-and this to your crate root:
-
-```rust
-extern crate parquet;
-```
-
-Example usage of reading data:
-
-```rust
-use std::fs::File;
-use std::path::Path;
-use parquet::file::reader::{FileReader, SerializedFileReader};
-
-let file = File::open(&Path::new("/path/to/file")).unwrap();
-let reader = SerializedFileReader::new(file).unwrap();
-let mut iter = reader.get_row_iter(None).unwrap();
-while let Some(record) = iter.next() {
- println!("{}", record);
-}
-```
-
-See [crate documentation](https://docs.rs/crate/parquet/5.0.0-SNAPSHOT) on available API.
-
-## Upgrading from versions prior to 4.0
-
-If you are upgrading from version 3.0 or previous of this crate, you
-likely need to change your code to use [`ConvertedType`] rather than
-[`LogicalType`] to preserve existing behaviour in your code.
-
-Version 2.4.0 of the Parquet format introduced a `LogicalType` to replace the existing `ConvertedType`.
-This crate used `parquet::basic::LogicalType` to map to the `ConvertedType`, but this has been renamed to `parquet::basic::ConvertedType` from version 4.0 of this crate.
-
-The `ConvertedType` is deprecated in the format, but is still written
-to preserve backward compatibility.
-It is preferred that `LogicalType` is used, as it supports nanosecond
-precision timestamps without using the deprecated `Int96` Parquet type.
-
-## Supported Parquet Version
-
-- Parquet-format 2.6.0
-
-To update Parquet format to a newer version, check if [parquet-format](https://github.com/sunchao/parquet-format-rs)
-version is available. Then simply update version of `parquet-format` crate in Cargo.toml.
-
-## Features
-
-- [x] All encodings supported
-- [x] All compression codecs supported
-- [x] Read support
- - [x] Primitive column value readers
- - [x] Row record reader
- - [x] Arrow record reader
-- [ ] Statistics support
-- [x] Write support
- - [x] Primitive column value writers
- - [ ] Row record writer
- - [x] Arrow record writer
-- [ ] Predicate pushdown
-- [x] Parquet format 2.6.0 support
-
-## Requirements
-
-Parquet requires LLVM. Our windows CI image includes LLVM but to build the libraries locally windows
-users will have to install LLVM. Follow [this](https://github.com/appveyor/ci/issues/2651) link for info.
-
-## Build
-
-Run `cargo build` or `cargo build --release` to build in release mode.
-Some features take advantage of SSE4.2 instructions, which can be
-enabled by adding `RUSTFLAGS="-C target-feature=+sse4.2"` before the
-`cargo build` command.
-
-## Test
-
-Run `cargo test` for unit tests. To also run tests related to the binaries, use `cargo test --features cli`.
-
-## Binaries
-
-The following binaries are provided (use `cargo install --features cli` to install them):
-
-- **parquet-schema** for printing Parquet file schema and metadata.
- `Usage: parquet-schema <file-path>`, where `file-path` is the path to a Parquet file. Use `-v/--verbose` flag
- to print full metadata or schema only (when not specified only schema will be printed).
-
-- **parquet-read** for reading records from a Parquet file.
- `Usage: parquet-read <file-path> [num-records]`, where `file-path` is the path to a Parquet file,
- and `num-records` is the number of records to read from a file (when not specified all records will
- be printed). Use `-j/--json` to print records in JSON lines format.
-
-- **parquet-rowcount** for reporting the number of records in one or more Parquet files.
- `Usage: parquet-rowcount <file-paths>...`, where `<file-paths>...` is a space separated list of one or more
- files to read.
-
-If you see `Library not loaded` error, please make sure `LD_LIBRARY_PATH` is set properly:
-
-```
-export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$(rustc --print sysroot)/lib
-```
-
-## Benchmarks
-
-Run `cargo bench` for benchmarks.
-
-## Docs
-
-To build documentation, run `cargo doc --no-deps`.
-To compile and view in the browser, run `cargo doc --no-deps --open`.
-
-## License
-
-Licensed under the Apache License, Version 2.0: http://www.apache.org/licenses/LICENSE-2.0.
diff --git a/parquet/benches/arrow_array_reader.rs b/parquet/benches/arrow_array_reader.rs
deleted file mode 100644
index 6e87512..0000000
--- a/parquet/benches/arrow_array_reader.rs
+++ /dev/null
@@ -1,766 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-use criterion::{criterion_group, criterion_main, Criterion};
-use parquet::util::{DataPageBuilder, DataPageBuilderImpl, InMemoryPageIterator};
-use parquet::{
- arrow::array_reader::ArrayReader,
- basic::Encoding,
- column::page::PageIterator,
- data_type::{ByteArrayType, Int32Type},
- schema::types::{ColumnDescPtr, SchemaDescPtr},
-};
-use std::{collections::VecDeque, sync::Arc};
-
-fn build_test_schema() -> SchemaDescPtr {
- use parquet::schema::{parser::parse_message_type, types::SchemaDescriptor};
- let message_type = "
- message test_schema {
- REQUIRED INT32 mandatory_int32_leaf;
- REPEATED Group test_mid_int32 {
- OPTIONAL INT32 optional_int32_leaf;
- }
- REQUIRED BYTE_ARRAY mandatory_string_leaf (UTF8);
- REPEATED Group test_mid_string {
- OPTIONAL BYTE_ARRAY optional_string_leaf (UTF8);
- }
- }
- ";
- parse_message_type(message_type)
- .map(|t| Arc::new(SchemaDescriptor::new(Arc::new(t))))
- .unwrap()
-}
-
-// test data params
-const NUM_ROW_GROUPS: usize = 1;
-const PAGES_PER_GROUP: usize = 2;
-const VALUES_PER_PAGE: usize = 10_000;
-const BATCH_SIZE: usize = 8192;
-
-use rand::{rngs::StdRng, Rng, SeedableRng};
-
-pub fn seedable_rng() -> StdRng {
- StdRng::seed_from_u64(42)
-}
-
-fn build_plain_encoded_int32_page_iterator(
- schema: SchemaDescPtr,
- column_desc: ColumnDescPtr,
- null_density: f32,
-) -> impl PageIterator + Clone {
- let max_def_level = column_desc.max_def_level();
- let max_rep_level = column_desc.max_rep_level();
- let rep_levels = vec![0; VALUES_PER_PAGE];
- let mut rng = seedable_rng();
- let mut pages: Vec<Vec<parquet::column::page::Page>> = Vec::new();
- let mut int32_value = 0;
- for _i in 0..NUM_ROW_GROUPS {
- let mut column_chunk_pages = Vec::new();
- for _j in 0..PAGES_PER_GROUP {
- // generate page
- let mut values = Vec::with_capacity(VALUES_PER_PAGE);
- let mut def_levels = Vec::with_capacity(VALUES_PER_PAGE);
- for _k in 0..VALUES_PER_PAGE {
- let def_level = if rng.gen::<f32>() < null_density {
- max_def_level - 1
- } else {
- max_def_level
- };
- if def_level == max_def_level {
- int32_value += 1;
- values.push(int32_value);
- }
- def_levels.push(def_level);
- }
- let mut page_builder =
- DataPageBuilderImpl::new(column_desc.clone(), values.len() as u32, true);
- page_builder.add_rep_levels(max_rep_level, &rep_levels);
- page_builder.add_def_levels(max_def_level, &def_levels);
- page_builder.add_values::<Int32Type>(Encoding::PLAIN, &values);
- column_chunk_pages.push(page_builder.consume());
- }
- pages.push(column_chunk_pages);
- }
-
- InMemoryPageIterator::new(schema, column_desc, pages)
-}
-
-fn build_dictionary_encoded_int32_page_iterator(
- schema: SchemaDescPtr,
- column_desc: ColumnDescPtr,
- null_density: f32,
-) -> impl PageIterator + Clone {
- use parquet::encoding::{DictEncoder, Encoder};
- let max_def_level = column_desc.max_def_level();
- let max_rep_level = column_desc.max_rep_level();
- let rep_levels = vec![0; VALUES_PER_PAGE];
- // generate 1% unique values
- const NUM_UNIQUE_VALUES: usize = VALUES_PER_PAGE / 100;
- let unique_values = (0..NUM_UNIQUE_VALUES)
- .map(|x| (x + 1) as i32)
- .collect::<Vec<_>>();
- let mut rng = seedable_rng();
- let mut pages: Vec<Vec<parquet::column::page::Page>> = Vec::new();
- for _i in 0..NUM_ROW_GROUPS {
- let mut column_chunk_pages = VecDeque::new();
- let mem_tracker = Arc::new(parquet::memory::MemTracker::new());
- let mut dict_encoder =
- DictEncoder::<Int32Type>::new(column_desc.clone(), mem_tracker);
- // add data pages
- for _j in 0..PAGES_PER_GROUP {
- // generate page
- let mut values = Vec::with_capacity(VALUES_PER_PAGE);
- let mut def_levels = Vec::with_capacity(VALUES_PER_PAGE);
- for _k in 0..VALUES_PER_PAGE {
- let def_level = if rng.gen::<f32>() < null_density {
- max_def_level - 1
- } else {
- max_def_level
- };
- if def_level == max_def_level {
- // select random value from list of unique values
- let int32_value = unique_values[rng.gen_range(0..NUM_UNIQUE_VALUES)];
- values.push(int32_value);
- }
- def_levels.push(def_level);
- }
- let mut page_builder =
- DataPageBuilderImpl::new(column_desc.clone(), values.len() as u32, true);
- page_builder.add_rep_levels(max_rep_level, &rep_levels);
- page_builder.add_def_levels(max_def_level, &def_levels);
- let _ = dict_encoder.put(&values);
- let indices = dict_encoder
- .write_indices()
- .expect("write_indices() should be OK");
- page_builder.add_indices(indices);
- column_chunk_pages.push_back(page_builder.consume());
- }
- // add dictionary page
- let dict = dict_encoder
- .write_dict()
- .expect("write_dict() should be OK");
- let dict_page = parquet::column::page::Page::DictionaryPage {
- buf: dict,
- num_values: dict_encoder.num_entries() as u32,
- encoding: Encoding::RLE_DICTIONARY,
- is_sorted: false,
- };
- column_chunk_pages.push_front(dict_page);
- pages.push(column_chunk_pages.into());
- }
-
- InMemoryPageIterator::new(schema, column_desc, pages)
-}
-
-fn build_plain_encoded_string_page_iterator(
- schema: SchemaDescPtr,
- column_desc: ColumnDescPtr,
- null_density: f32,
-) -> impl PageIterator + Clone {
- let max_def_level = column_desc.max_def_level();
- let max_rep_level = column_desc.max_rep_level();
- let rep_levels = vec![0; VALUES_PER_PAGE];
- let mut rng = seedable_rng();
- let mut pages: Vec<Vec<parquet::column::page::Page>> = Vec::new();
- for i in 0..NUM_ROW_GROUPS {
- let mut column_chunk_pages = Vec::new();
- for j in 0..PAGES_PER_GROUP {
- // generate page
- let mut values = Vec::with_capacity(VALUES_PER_PAGE);
- let mut def_levels = Vec::with_capacity(VALUES_PER_PAGE);
- for k in 0..VALUES_PER_PAGE {
- let def_level = if rng.gen::<f32>() < null_density {
- max_def_level - 1
- } else {
- max_def_level
- };
- if def_level == max_def_level {
- let string_value =
- format!("Test value {}, row group: {}, page: {}", k, i, j);
- values
- .push(parquet::data_type::ByteArray::from(string_value.as_str()));
- }
- def_levels.push(def_level);
- }
- let mut page_builder =
- DataPageBuilderImpl::new(column_desc.clone(), values.len() as u32, true);
- page_builder.add_rep_levels(max_rep_level, &rep_levels);
- page_builder.add_def_levels(max_def_level, &def_levels);
- page_builder.add_values::<ByteArrayType>(Encoding::PLAIN, &values);
- column_chunk_pages.push(page_builder.consume());
- }
- pages.push(column_chunk_pages);
- }
-
- InMemoryPageIterator::new(schema, column_desc, pages)
-}
-
-fn build_dictionary_encoded_string_page_iterator(
- schema: SchemaDescPtr,
- column_desc: ColumnDescPtr,
- null_density: f32,
-) -> impl PageIterator + Clone {
- use parquet::encoding::{DictEncoder, Encoder};
- let max_def_level = column_desc.max_def_level();
- let max_rep_level = column_desc.max_rep_level();
- let rep_levels = vec![0; VALUES_PER_PAGE];
- // generate 1% unique values
- const NUM_UNIQUE_VALUES: usize = VALUES_PER_PAGE / 100;
- let unique_values = (0..NUM_UNIQUE_VALUES)
- .map(|x| format!("Dictionary value {}", x))
- .collect::<Vec<_>>();
- let mut rng = seedable_rng();
- let mut pages: Vec<Vec<parquet::column::page::Page>> = Vec::new();
- for _i in 0..NUM_ROW_GROUPS {
- let mut column_chunk_pages = VecDeque::new();
- let mem_tracker = Arc::new(parquet::memory::MemTracker::new());
- let mut dict_encoder =
- DictEncoder::<ByteArrayType>::new(column_desc.clone(), mem_tracker);
- // add data pages
- for _j in 0..PAGES_PER_GROUP {
- // generate page
- let mut values = Vec::with_capacity(VALUES_PER_PAGE);
- let mut def_levels = Vec::with_capacity(VALUES_PER_PAGE);
- for _k in 0..VALUES_PER_PAGE {
- let def_level = if rng.gen::<f32>() < null_density {
- max_def_level - 1
- } else {
- max_def_level
- };
- if def_level == max_def_level {
- // select random value from list of unique values
- let string_value =
- unique_values[rng.gen_range(0..NUM_UNIQUE_VALUES)].as_str();
- values.push(parquet::data_type::ByteArray::from(string_value));
- }
- def_levels.push(def_level);
- }
- let mut page_builder =
- DataPageBuilderImpl::new(column_desc.clone(), values.len() as u32, true);
- page_builder.add_rep_levels(max_rep_level, &rep_levels);
- page_builder.add_def_levels(max_def_level, &def_levels);
- let _ = dict_encoder.put(&values);
- let indices = dict_encoder
- .write_indices()
- .expect("write_indices() should be OK");
- page_builder.add_indices(indices);
- column_chunk_pages.push_back(page_builder.consume());
- }
- // add dictionary page
- let dict = dict_encoder
- .write_dict()
- .expect("write_dict() should be OK");
- let dict_page = parquet::column::page::Page::DictionaryPage {
- buf: dict,
- num_values: dict_encoder.num_entries() as u32,
- encoding: Encoding::RLE_DICTIONARY,
- is_sorted: false,
- };
- column_chunk_pages.push_front(dict_page);
- pages.push(column_chunk_pages.into());
- }
-
- InMemoryPageIterator::new(schema, column_desc, pages)
-}
-
-fn bench_array_reader(mut array_reader: impl ArrayReader) -> usize {
- // test procedure: read data in batches of 8192 until no more data
- let mut total_count = 0;
- loop {
- let array = array_reader.next_batch(BATCH_SIZE);
- let array_len = array.unwrap().len();
- total_count += array_len;
- if array_len < BATCH_SIZE {
- break;
- }
- }
- total_count
-}
-
-fn create_int32_arrow_array_reader(
- page_iterator: impl PageIterator + 'static,
- column_desc: ColumnDescPtr,
-) -> impl ArrayReader {
- use parquet::arrow::arrow_array_reader::{ArrowArrayReader, PrimitiveArrayConverter};
- let converter = PrimitiveArrayConverter::<arrow::datatypes::Int32Type>::new();
- ArrowArrayReader::try_new(page_iterator, column_desc, converter, None).unwrap()
-}
-
-fn create_int32_primitive_array_reader(
- page_iterator: impl PageIterator + 'static,
- column_desc: ColumnDescPtr,
-) -> impl ArrayReader {
- use parquet::arrow::array_reader::PrimitiveArrayReader;
- PrimitiveArrayReader::<Int32Type>::new(Box::new(page_iterator), column_desc, None)
- .unwrap()
-}
-
-fn create_string_arrow_array_reader(
- page_iterator: impl PageIterator + 'static,
- column_desc: ColumnDescPtr,
-) -> impl ArrayReader {
- use parquet::arrow::arrow_array_reader::{ArrowArrayReader, StringArrayConverter};
- let converter = StringArrayConverter::new();
- ArrowArrayReader::try_new(page_iterator, column_desc, converter, None).unwrap()
-}
-
-fn create_string_complex_array_reader(
- page_iterator: impl PageIterator + 'static,
- column_desc: ColumnDescPtr,
-) -> impl ArrayReader {
- use parquet::arrow::array_reader::ComplexObjectArrayReader;
- use parquet::arrow::converter::{Utf8ArrayConverter, Utf8Converter};
- let converter = Utf8Converter::new(Utf8ArrayConverter {});
- ComplexObjectArrayReader::<parquet::data_type::ByteArrayType, Utf8Converter>::new(
- Box::new(page_iterator),
- column_desc,
- converter,
- None,
- )
- .unwrap()
-}
-
-fn add_benches(c: &mut Criterion) {
- const EXPECTED_VALUE_COUNT: usize =
- NUM_ROW_GROUPS * PAGES_PER_GROUP * VALUES_PER_PAGE;
- let mut group = c.benchmark_group("arrow_array_reader");
-
- let mut count: usize = 0;
-
- let schema = build_test_schema();
- let mandatory_int32_column_desc = schema.column(0);
- let optional_int32_column_desc = schema.column(1);
- let mandatory_string_column_desc = schema.column(2);
- // println!("mandatory_string_column_desc: {:?}", mandatory_string_column_desc);
- let optional_string_column_desc = schema.column(3);
- // println!("optional_string_column_desc: {:?}", optional_string_column_desc);
-
- // primitive / int32 benchmarks
- // =============================
-
- // int32, plain encoded, no NULLs
- let plain_int32_no_null_data = build_plain_encoded_int32_page_iterator(
- schema.clone(),
- mandatory_int32_column_desc.clone(),
- 0.0,
- );
- group.bench_function(
- "read Int32Array, plain encoded, mandatory, no NULLs - old",
- |b| {
- b.iter(|| {
- let array_reader = create_int32_primitive_array_reader(
- plain_int32_no_null_data.clone(),
- mandatory_int32_column_desc.clone(),
- );
- count = bench_array_reader(array_reader);
- })
- },
- );
- assert_eq!(count, EXPECTED_VALUE_COUNT);
-
- group.bench_function(
- "read Int32Array, plain encoded, mandatory, no NULLs - new",
- |b| {
- b.iter(|| {
- let array_reader = create_int32_arrow_array_reader(
- plain_int32_no_null_data.clone(),
- mandatory_int32_column_desc.clone(),
- );
- count = bench_array_reader(array_reader);
- })
- },
- );
- assert_eq!(count, EXPECTED_VALUE_COUNT);
-
- let plain_int32_no_null_data = build_plain_encoded_int32_page_iterator(
- schema.clone(),
- optional_int32_column_desc.clone(),
- 0.0,
- );
- group.bench_function(
- "read Int32Array, plain encoded, optional, no NULLs - old",
- |b| {
- b.iter(|| {
- let array_reader = create_int32_primitive_array_reader(
- plain_int32_no_null_data.clone(),
- optional_int32_column_desc.clone(),
- );
- count = bench_array_reader(array_reader);
- })
- },
- );
- assert_eq!(count, EXPECTED_VALUE_COUNT);
-
- group.bench_function(
- "read Int32Array, plain encoded, optional, no NULLs - new",
- |b| {
- b.iter(|| {
- let array_reader = create_int32_arrow_array_reader(
- plain_int32_no_null_data.clone(),
- optional_int32_column_desc.clone(),
- );
- count = bench_array_reader(array_reader);
- })
- },
- );
- assert_eq!(count, EXPECTED_VALUE_COUNT);
-
- // int32, plain encoded, half NULLs
- let plain_int32_half_null_data = build_plain_encoded_int32_page_iterator(
- schema.clone(),
- optional_int32_column_desc.clone(),
- 0.5,
- );
- group.bench_function(
- "read Int32Array, plain encoded, optional, half NULLs - old",
- |b| {
- b.iter(|| {
- let array_reader = create_int32_primitive_array_reader(
- plain_int32_half_null_data.clone(),
- optional_int32_column_desc.clone(),
- );
- count = bench_array_reader(array_reader);
- })
- },
- );
- assert_eq!(count, EXPECTED_VALUE_COUNT);
-
- group.bench_function(
- "read Int32Array, plain encoded, optional, half NULLs - new",
- |b| {
- b.iter(|| {
- let array_reader = create_int32_arrow_array_reader(
- plain_int32_half_null_data.clone(),
- optional_int32_column_desc.clone(),
- );
- count = bench_array_reader(array_reader);
- })
- },
- );
- assert_eq!(count, EXPECTED_VALUE_COUNT);
-
- // int32, dictionary encoded, no NULLs
- let dictionary_int32_no_null_data = build_dictionary_encoded_int32_page_iterator(
- schema.clone(),
- mandatory_int32_column_desc.clone(),
- 0.0,
- );
- group.bench_function(
- "read Int32Array, dictionary encoded, mandatory, no NULLs - old",
- |b| {
- b.iter(|| {
- let array_reader = create_int32_primitive_array_reader(
- dictionary_int32_no_null_data.clone(),
- mandatory_int32_column_desc.clone(),
- );
- count = bench_array_reader(array_reader);
- })
- },
- );
- assert_eq!(count, EXPECTED_VALUE_COUNT);
-
- group.bench_function(
- "read Int32Array, dictionary encoded, mandatory, no NULLs - new",
- |b| {
- b.iter(|| {
- let array_reader = create_int32_arrow_array_reader(
- dictionary_int32_no_null_data.clone(),
- mandatory_int32_column_desc.clone(),
- );
- count = bench_array_reader(array_reader);
- })
- },
- );
- assert_eq!(count, EXPECTED_VALUE_COUNT);
-
- let dictionary_int32_no_null_data = build_dictionary_encoded_int32_page_iterator(
- schema.clone(),
- optional_int32_column_desc.clone(),
- 0.0,
- );
- group.bench_function(
- "read Int32Array, dictionary encoded, optional, no NULLs - old",
- |b| {
- b.iter(|| {
- let array_reader = create_int32_primitive_array_reader(
- dictionary_int32_no_null_data.clone(),
- optional_int32_column_desc.clone(),
- );
- count = bench_array_reader(array_reader);
- })
- },
- );
- assert_eq!(count, EXPECTED_VALUE_COUNT);
-
- group.bench_function(
- "read Int32Array, dictionary encoded, optional, no NULLs - new",
- |b| {
- b.iter(|| {
- let array_reader = create_int32_arrow_array_reader(
- dictionary_int32_no_null_data.clone(),
- optional_int32_column_desc.clone(),
- );
- count = bench_array_reader(array_reader);
- })
- },
- );
- assert_eq!(count, EXPECTED_VALUE_COUNT);
-
- // int32, dictionary encoded, half NULLs
- let dictionary_int32_half_null_data = build_dictionary_encoded_int32_page_iterator(
- schema.clone(),
- optional_int32_column_desc.clone(),
- 0.5,
- );
- group.bench_function(
- "read Int32Array, dictionary encoded, optional, half NULLs - old",
- |b| {
- b.iter(|| {
- let array_reader = create_int32_primitive_array_reader(
- dictionary_int32_half_null_data.clone(),
- optional_int32_column_desc.clone(),
- );
- count = bench_array_reader(array_reader);
- })
- },
- );
- assert_eq!(count, EXPECTED_VALUE_COUNT);
-
- group.bench_function(
- "read Int32Array, dictionary encoded, optional, half NULLs - new",
- |b| {
- b.iter(|| {
- let array_reader = create_int32_arrow_array_reader(
- dictionary_int32_half_null_data.clone(),
- optional_int32_column_desc.clone(),
- );
- count = bench_array_reader(array_reader);
- })
- },
- );
- assert_eq!(count, EXPECTED_VALUE_COUNT);
-
- // string benchmarks
- //==============================
-
- // string, plain encoded, no NULLs
- let plain_string_no_null_data = build_plain_encoded_string_page_iterator(
- schema.clone(),
- mandatory_string_column_desc.clone(),
- 0.0,
- );
- group.bench_function(
- "read StringArray, plain encoded, mandatory, no NULLs - old",
- |b| {
- b.iter(|| {
- let array_reader = create_string_complex_array_reader(
- plain_string_no_null_data.clone(),
- mandatory_string_column_desc.clone(),
- );
- count = bench_array_reader(array_reader);
- })
- },
- );
- assert_eq!(count, EXPECTED_VALUE_COUNT);
-
- group.bench_function(
- "read StringArray, plain encoded, mandatory, no NULLs - new",
- |b| {
- b.iter(|| {
- let array_reader = create_string_arrow_array_reader(
- plain_string_no_null_data.clone(),
- mandatory_string_column_desc.clone(),
- );
- count = bench_array_reader(array_reader);
- })
- },
- );
- assert_eq!(count, EXPECTED_VALUE_COUNT);
-
- let plain_string_no_null_data = build_plain_encoded_string_page_iterator(
- schema.clone(),
- optional_string_column_desc.clone(),
- 0.0,
- );
- group.bench_function(
- "read StringArray, plain encoded, optional, no NULLs - old",
- |b| {
- b.iter(|| {
- let array_reader = create_string_complex_array_reader(
- plain_string_no_null_data.clone(),
- optional_string_column_desc.clone(),
- );
- count = bench_array_reader(array_reader);
- })
- },
- );
- assert_eq!(count, EXPECTED_VALUE_COUNT);
-
- group.bench_function(
- "read StringArray, plain encoded, optional, no NULLs - new",
- |b| {
- b.iter(|| {
- let array_reader = create_string_arrow_array_reader(
- plain_string_no_null_data.clone(),
- optional_string_column_desc.clone(),
- );
- count = bench_array_reader(array_reader);
- })
- },
- );
- assert_eq!(count, EXPECTED_VALUE_COUNT);
-
- // string, plain encoded, half NULLs
- let plain_string_half_null_data = build_plain_encoded_string_page_iterator(
- schema.clone(),
- optional_string_column_desc.clone(),
- 0.5,
- );
- group.bench_function(
- "read StringArray, plain encoded, optional, half NULLs - old",
- |b| {
- b.iter(|| {
- let array_reader = create_string_complex_array_reader(
- plain_string_half_null_data.clone(),
- optional_string_column_desc.clone(),
- );
- count = bench_array_reader(array_reader);
- })
- },
- );
- assert_eq!(count, EXPECTED_VALUE_COUNT);
-
- group.bench_function(
- "read StringArray, plain encoded, optional, half NULLs - new",
- |b| {
- b.iter(|| {
- let array_reader = create_string_arrow_array_reader(
- plain_string_half_null_data.clone(),
- optional_string_column_desc.clone(),
- );
- count = bench_array_reader(array_reader);
- })
- },
- );
- assert_eq!(count, EXPECTED_VALUE_COUNT);
-
- // string, dictionary encoded, no NULLs
- let dictionary_string_no_null_data = build_dictionary_encoded_string_page_iterator(
- schema.clone(),
- mandatory_string_column_desc.clone(),
- 0.0,
- );
- group.bench_function(
- "read StringArray, dictionary encoded, mandatory, no NULLs - old",
- |b| {
- b.iter(|| {
- let array_reader = create_string_complex_array_reader(
- dictionary_string_no_null_data.clone(),
- mandatory_string_column_desc.clone(),
- );
- count = bench_array_reader(array_reader);
- })
- },
- );
- assert_eq!(count, EXPECTED_VALUE_COUNT);
-
- group.bench_function(
- "read StringArray, dictionary encoded, mandatory, no NULLs - new",
- |b| {
- b.iter(|| {
- let array_reader = create_string_arrow_array_reader(
- dictionary_string_no_null_data.clone(),
- mandatory_string_column_desc.clone(),
- );
- count = bench_array_reader(array_reader);
- })
- },
- );
- assert_eq!(count, EXPECTED_VALUE_COUNT);
-
- let dictionary_string_no_null_data = build_dictionary_encoded_string_page_iterator(
- schema.clone(),
- optional_string_column_desc.clone(),
- 0.0,
- );
- group.bench_function(
- "read StringArray, dictionary encoded, optional, no NULLs - old",
- |b| {
- b.iter(|| {
- let array_reader = create_string_complex_array_reader(
- dictionary_string_no_null_data.clone(),
- optional_string_column_desc.clone(),
- );
- count = bench_array_reader(array_reader);
- })
- },
- );
- assert_eq!(count, EXPECTED_VALUE_COUNT);
-
- group.bench_function(
- "read StringArray, dictionary encoded, optional, no NULLs - new",
- |b| {
- b.iter(|| {
- let array_reader = create_string_arrow_array_reader(
- dictionary_string_no_null_data.clone(),
- optional_string_column_desc.clone(),
- );
- count = bench_array_reader(array_reader);
- })
- },
- );
- assert_eq!(count, EXPECTED_VALUE_COUNT);
-
- // string, dictionary encoded, half NULLs
- let dictionary_string_half_null_data = build_dictionary_encoded_string_page_iterator(
- schema,
- optional_string_column_desc.clone(),
- 0.5,
- );
- group.bench_function(
- "read StringArray, dictionary encoded, optional, half NULLs - old",
- |b| {
- b.iter(|| {
- let array_reader = create_string_complex_array_reader(
- dictionary_string_half_null_data.clone(),
- optional_string_column_desc.clone(),
- );
- count = bench_array_reader(array_reader);
- })
- },
- );
- assert_eq!(count, EXPECTED_VALUE_COUNT);
-
- group.bench_function(
- "read StringArray, dictionary encoded, optional, half NULLs - new",
- |b| {
- b.iter(|| {
- let array_reader = create_string_arrow_array_reader(
- dictionary_string_half_null_data.clone(),
- optional_string_column_desc.clone(),
- );
- count = bench_array_reader(array_reader);
- })
- },
- );
- assert_eq!(count, EXPECTED_VALUE_COUNT);
-
- group.finish();
-}
-
-criterion_group!(benches, add_benches);
-criterion_main!(benches);
diff --git a/parquet/benches/arrow_writer.rs b/parquet/benches/arrow_writer.rs
deleted file mode 100644
index 069ed39..0000000
--- a/parquet/benches/arrow_writer.rs
+++ /dev/null
@@ -1,202 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#[macro_use]
-extern crate criterion;
-use criterion::{Criterion, Throughput};
-
-extern crate arrow;
-extern crate parquet;
-
-use std::sync::Arc;
-
-use arrow::datatypes::*;
-use arrow::{record_batch::RecordBatch, util::data_gen::*};
-use parquet::{
- arrow::ArrowWriter, errors::Result, file::writer::InMemoryWriteableCursor,
-};
-
-fn create_primitive_bench_batch(
- size: usize,
- null_density: f32,
- true_density: f32,
-) -> Result<RecordBatch> {
- let fields = vec![
- Field::new("_1", DataType::Int8, true),
- Field::new("_2", DataType::Int16, true),
- Field::new("_3", DataType::Int32, true),
- Field::new("_4", DataType::Int64, true),
- Field::new("_5", DataType::UInt8, true),
- Field::new("_6", DataType::UInt16, true),
- Field::new("_7", DataType::UInt32, true),
- Field::new("_8", DataType::UInt64, true),
- Field::new("_9", DataType::Float32, true),
- Field::new("_10", DataType::Float64, true),
- Field::new("_11", DataType::Date32, true),
- Field::new("_12", DataType::Date64, true),
- Field::new("_13", DataType::Time32(TimeUnit::Second), true),
- Field::new("_14", DataType::Time32(TimeUnit::Millisecond), true),
- Field::new("_15", DataType::Time64(TimeUnit::Microsecond), true),
- Field::new("_16", DataType::Time64(TimeUnit::Nanosecond), true),
- Field::new("_17", DataType::Utf8, true),
- Field::new("_18", DataType::LargeUtf8, true),
- Field::new("_19", DataType::Boolean, true),
- ];
- let schema = Schema::new(fields);
- Ok(create_random_batch(
- Arc::new(schema),
- size,
- null_density,
- true_density,
- )?)
-}
-
-fn _create_nested_bench_batch(
- size: usize,
- null_density: f32,
- true_density: f32,
-) -> Result<RecordBatch> {
- let fields = vec![
- Field::new(
- "_1",
- DataType::Struct(vec![
- Field::new("_1", DataType::Int8, true),
- Field::new(
- "_2",
- DataType::Struct(vec![
- Field::new("_1", DataType::Int8, true),
- Field::new(
- "_1",
- DataType::Struct(vec![
- Field::new("_1", DataType::Int8, true),
- Field::new("_2", DataType::Utf8, true),
- ]),
- true,
- ),
- Field::new("_2", DataType::UInt8, true),
- ]),
- true,
- ),
- ]),
- true,
- ),
- Field::new(
- "_2",
- DataType::LargeList(Box::new(Field::new(
- "item",
- DataType::List(Box::new(Field::new(
- "item",
- DataType::Struct(vec![
- Field::new(
- "_1",
- DataType::Struct(vec![
- Field::new("_1", DataType::Int8, true),
- Field::new("_2", DataType::Int16, true),
- Field::new("_3", DataType::Int32, true),
- ]),
- true,
- ),
- Field::new(
- "_2",
- DataType::List(Box::new(Field::new(
- "",
- DataType::FixedSizeBinary(2),
- true,
- ))),
- true,
- ),
- ]),
- true,
- ))),
- true,
- ))),
- true,
- ),
- ];
- let schema = Schema::new(fields);
- Ok(create_random_batch(
- Arc::new(schema),
- size,
- null_density,
- true_density,
- )?)
-}
-
-#[inline]
-fn write_batch(batch: &RecordBatch) -> Result<()> {
- // Write batch to an in-memory writer
- let cursor = InMemoryWriteableCursor::default();
- let mut writer = ArrowWriter::try_new(cursor, batch.schema(), None)?;
-
- writer.write(&batch)?;
- writer.close()?;
- Ok(())
-}
-
-fn bench_primitive_writer(c: &mut Criterion) {
- let batch = create_primitive_bench_batch(1024, 0.25, 0.75).unwrap();
- let mut group = c.benchmark_group("write_batch primitive");
- group.throughput(Throughput::Bytes(
- batch
- .columns()
- .iter()
- .map(|f| f.get_array_memory_size() as u64)
- .sum(),
- ));
- group.bench_function("1024 values", |b| b.iter(|| write_batch(&batch).unwrap()));
-
- let batch = create_primitive_bench_batch(4096, 0.25, 0.75).unwrap();
- group.throughput(Throughput::Bytes(
- batch
- .columns()
- .iter()
- .map(|f| f.get_array_memory_size() as u64)
- .sum(),
- ));
- group.bench_function("4096 values", |b| b.iter(|| write_batch(&batch).unwrap()));
-
- group.finish();
-}
-
-// This bench triggers a write error, it is ignored for now
-fn _bench_nested_writer(c: &mut Criterion) {
- let batch = _create_nested_bench_batch(1024, 0.25, 0.75).unwrap();
- let mut group = c.benchmark_group("write_batch nested");
- group.throughput(Throughput::Bytes(
- batch
- .columns()
- .iter()
- .map(|f| f.get_array_memory_size() as u64)
- .sum(),
- ));
- group.bench_function("1024 values", |b| b.iter(|| write_batch(&batch).unwrap()));
-
- let batch = create_primitive_bench_batch(4096, 0.25, 0.75).unwrap();
- group.throughput(Throughput::Bytes(
- batch
- .columns()
- .iter()
- .map(|f| f.get_array_memory_size() as u64)
- .sum(),
- ));
- group.bench_function("4096 values", |b| b.iter(|| write_batch(&batch).unwrap()));
-
- group.finish();
-}
-
-criterion_group!(benches, bench_primitive_writer);
-criterion_main!(benches);
diff --git a/parquet/build.rs b/parquet/build.rs
deleted file mode 100644
index b42b2a4..0000000
--- a/parquet/build.rs
+++ /dev/null
@@ -1,43 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-use std::process::Command;
-
-fn main() {
- // Set Parquet version, build hash and "created by" string.
- let version = env!("CARGO_PKG_VERSION");
- let mut created_by = format!("parquet-rs version {}", version);
- if let Ok(git_hash) = run(Command::new("git").arg("rev-parse").arg("HEAD")) {
- created_by.push_str(format!(" (build {})", git_hash).as_str());
- println!("cargo:rustc-env=PARQUET_BUILD={}", git_hash);
- }
- println!("cargo:rustc-env=PARQUET_VERSION={}", version);
- println!("cargo:rustc-env=PARQUET_CREATED_BY={}", created_by);
-}
-
-/// Runs command and returns either content of stdout for successful execution,
-/// or an error message otherwise.
-fn run(command: &mut Command) -> Result<String, String> {
- println!("Running: `{:?}`", command);
- match command.output() {
- Ok(ref output) if output.status.success() => {
- Ok(String::from_utf8_lossy(&output.stdout).trim().to_string())
- }
- Ok(ref output) => Err(format!("Failed: `{:?}` ({})", command, output.status)),
- Err(error) => Err(format!("Failed: `{:?}` ({})", command, error)),
- }
-}
diff --git a/parquet/src/arrow/array_reader.rs b/parquet/src/arrow/array_reader.rs
deleted file mode 100644
index bd57cf3..0000000
--- a/parquet/src/arrow/array_reader.rs
+++ /dev/null
@@ -1,2556 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-use std::cmp::{max, min};
-use std::collections::{HashMap, HashSet};
-use std::marker::PhantomData;
-use std::mem::size_of;
-use std::result::Result::Ok;
-use std::sync::Arc;
-use std::vec::Vec;
-
-use arrow::array::{
- new_empty_array, Array, ArrayData, ArrayDataBuilder, ArrayRef, BinaryArray,
- BinaryBuilder, BooleanArray, BooleanBufferBuilder, BooleanBuilder, DecimalBuilder,
- FixedSizeBinaryArray, FixedSizeBinaryBuilder, GenericListArray, Int16BufferBuilder,
- Int32Array, Int64Array, OffsetSizeTrait, PrimitiveArray, PrimitiveBuilder,
- StringArray, StringBuilder, StructArray,
-};
-use arrow::buffer::{Buffer, MutableBuffer};
-use arrow::datatypes::{
- ArrowPrimitiveType, BooleanType as ArrowBooleanType, DataType as ArrowType,
- Date32Type as ArrowDate32Type, Date64Type as ArrowDate64Type,
- DurationMicrosecondType as ArrowDurationMicrosecondType,
- DurationMillisecondType as ArrowDurationMillisecondType,
- DurationNanosecondType as ArrowDurationNanosecondType,
- DurationSecondType as ArrowDurationSecondType, Field,
- Float32Type as ArrowFloat32Type, Float64Type as ArrowFloat64Type,
- Int16Type as ArrowInt16Type, Int32Type as ArrowInt32Type,
- Int64Type as ArrowInt64Type, Int8Type as ArrowInt8Type, IntervalUnit, Schema,
- Time32MillisecondType as ArrowTime32MillisecondType,
- Time32SecondType as ArrowTime32SecondType,
- Time64MicrosecondType as ArrowTime64MicrosecondType,
- Time64NanosecondType as ArrowTime64NanosecondType, TimeUnit as ArrowTimeUnit,
- TimestampMicrosecondType as ArrowTimestampMicrosecondType,
- TimestampMillisecondType as ArrowTimestampMillisecondType,
- TimestampNanosecondType as ArrowTimestampNanosecondType,
- TimestampSecondType as ArrowTimestampSecondType, ToByteSlice,
- UInt16Type as ArrowUInt16Type, UInt32Type as ArrowUInt32Type,
- UInt64Type as ArrowUInt64Type, UInt8Type as ArrowUInt8Type,
-};
-use arrow::util::bit_util;
-
-use crate::arrow::converter::{
- BinaryArrayConverter, BinaryConverter, Converter, DecimalArrayConverter,
- DecimalConverter, FixedLenBinaryConverter, FixedSizeArrayConverter,
- Int96ArrayConverter, Int96Converter, IntervalDayTimeArrayConverter,
- IntervalDayTimeConverter, IntervalYearMonthArrayConverter,
- IntervalYearMonthConverter, LargeBinaryArrayConverter, LargeBinaryConverter,
- LargeUtf8ArrayConverter, LargeUtf8Converter,
-};
-use crate::arrow::record_reader::RecordReader;
-use crate::arrow::schema::parquet_to_arrow_field;
-use crate::basic::{ConvertedType, Repetition, Type as PhysicalType};
-use crate::column::page::PageIterator;
-use crate::column::reader::ColumnReaderImpl;
-use crate::data_type::{
- BoolType, ByteArrayType, DataType, DoubleType, FixedLenByteArrayType, FloatType,
- Int32Type, Int64Type, Int96Type,
-};
-use crate::errors::{ParquetError, ParquetError::ArrowError, Result};
-use crate::file::reader::{FilePageIterator, FileReader};
-use crate::schema::types::{
- ColumnDescPtr, ColumnDescriptor, ColumnPath, SchemaDescPtr, Type, TypePtr,
-};
-use crate::schema::visitor::TypeVisitor;
-use std::any::Any;
-
-/// Array reader reads parquet data into arrow array.
-pub trait ArrayReader {
- fn as_any(&self) -> &dyn Any;
-
- /// Returns the arrow type of this array reader.
- fn get_data_type(&self) -> &ArrowType;
-
- /// Reads at most `batch_size` records into an arrow array and return it.
- fn next_batch(&mut self, batch_size: usize) -> Result<ArrayRef>;
-
- /// Returns the definition levels of data from last call of `next_batch`.
- /// The result is used by parent array reader to calculate its own definition
- /// levels and repetition levels, so that its parent can calculate null bitmap.
- fn get_def_levels(&self) -> Option<&[i16]>;
-
- /// Return the repetition levels of data from last call of `next_batch`.
- /// The result is used by parent array reader to calculate its own definition
- /// levels and repetition levels, so that its parent can calculate null bitmap.
- fn get_rep_levels(&self) -> Option<&[i16]>;
-}
-
-/// A NullArrayReader reads Parquet columns stored as null int32s with an Arrow
-/// NullArray type.
-pub struct NullArrayReader<T: DataType> {
- data_type: ArrowType,
- pages: Box<dyn PageIterator>,
- def_levels_buffer: Option<Buffer>,
- rep_levels_buffer: Option<Buffer>,
- column_desc: ColumnDescPtr,
- record_reader: RecordReader<T>,
- _type_marker: PhantomData<T>,
-}
-
-impl<T: DataType> NullArrayReader<T> {
- /// Construct null array reader.
- pub fn new(
- mut pages: Box<dyn PageIterator>,
- column_desc: ColumnDescPtr,
- ) -> Result<Self> {
- let mut record_reader = RecordReader::<T>::new(column_desc.clone());
- if let Some(page_reader) = pages.next() {
- record_reader.set_page_reader(page_reader?)?;
- }
-
- Ok(Self {
- data_type: ArrowType::Null,
- pages,
- def_levels_buffer: None,
- rep_levels_buffer: None,
- column_desc,
- record_reader,
- _type_marker: PhantomData,
- })
- }
-}
-
-/// Implementation of primitive array reader.
-impl<T: DataType> ArrayReader for NullArrayReader<T> {
- fn as_any(&self) -> &dyn Any {
- self
- }
-
- /// Returns data type of primitive array.
- fn get_data_type(&self) -> &ArrowType {
- &self.data_type
- }
-
- /// Reads at most `batch_size` records into array.
- fn next_batch(&mut self, batch_size: usize) -> Result<ArrayRef> {
- let mut records_read = 0usize;
- while records_read < batch_size {
- let records_to_read = batch_size - records_read;
-
- // NB can be 0 if at end of page
- let records_read_once = self.record_reader.read_records(records_to_read)?;
- records_read += records_read_once;
-
- // Record reader exhausted
- if records_read_once < records_to_read {
- if let Some(page_reader) = self.pages.next() {
- // Read from new page reader
- self.record_reader.set_page_reader(page_reader?)?;
- } else {
- // Page reader also exhausted
- break;
- }
- }
- }
-
- // convert to arrays
- let array = arrow::array::NullArray::new(records_read);
-
- // save definition and repetition buffers
- self.def_levels_buffer = self.record_reader.consume_def_levels()?;
- self.rep_levels_buffer = self.record_reader.consume_rep_levels()?;
- self.record_reader.reset();
- Ok(Arc::new(array))
- }
-
- fn get_def_levels(&self) -> Option<&[i16]> {
- self.def_levels_buffer
- .as_ref()
- .map(|buf| unsafe { buf.typed_data() })
- }
-
- fn get_rep_levels(&self) -> Option<&[i16]> {
- self.rep_levels_buffer
- .as_ref()
- .map(|buf| unsafe { buf.typed_data() })
- }
-}
-
-/// Primitive array readers are leaves of array reader tree. They accept page iterator
-/// and read them into primitive arrays.
-pub struct PrimitiveArrayReader<T: DataType> {
- data_type: ArrowType,
- pages: Box<dyn PageIterator>,
- def_levels_buffer: Option<Buffer>,
- rep_levels_buffer: Option<Buffer>,
- column_desc: ColumnDescPtr,
- record_reader: RecordReader<T>,
- _type_marker: PhantomData<T>,
-}
-
-impl<T: DataType> PrimitiveArrayReader<T> {
- /// Construct primitive array reader.
- pub fn new(
- mut pages: Box<dyn PageIterator>,
- column_desc: ColumnDescPtr,
- arrow_type: Option<ArrowType>,
- ) -> Result<Self> {
- // Check if Arrow type is specified, else create it from Parquet type
- let data_type = match arrow_type {
- Some(t) => t,
- None => parquet_to_arrow_field(column_desc.as_ref())?
- .data_type()
- .clone(),
- };
-
- let mut record_reader = RecordReader::<T>::new(column_desc.clone());
- if let Some(page_reader) = pages.next() {
- record_reader.set_page_reader(page_reader?)?;
- }
-
- Ok(Self {
- data_type,
- pages,
- def_levels_buffer: None,
- rep_levels_buffer: None,
- column_desc,
- record_reader,
- _type_marker: PhantomData,
- })
- }
-}
-
-/// Implementation of primitive array reader.
-impl<T: DataType> ArrayReader for PrimitiveArrayReader<T> {
- fn as_any(&self) -> &dyn Any {
- self
- }
-
- /// Returns data type of primitive array.
- fn get_data_type(&self) -> &ArrowType {
- &self.data_type
- }
-
- /// Reads at most `batch_size` records into array.
- fn next_batch(&mut self, batch_size: usize) -> Result<ArrayRef> {
- let mut records_read = 0usize;
- while records_read < batch_size {
- let records_to_read = batch_size - records_read;
-
- // NB can be 0 if at end of page
- let records_read_once = self.record_reader.read_records(records_to_read)?;
- records_read += records_read_once;
-
- // Record reader exhausted
- if records_read_once < records_to_read {
- if let Some(page_reader) = self.pages.next() {
- // Read from new page reader
- self.record_reader.set_page_reader(page_reader?)?;
- } else {
- // Page reader also exhausted
- break;
- }
- }
- }
-
- let target_type = self.get_data_type().clone();
- let arrow_data_type = match T::get_physical_type() {
- PhysicalType::BOOLEAN => ArrowBooleanType::DATA_TYPE,
- PhysicalType::INT32 => {
- match target_type {
- ArrowType::UInt32 => {
- // follow C++ implementation and use overflow/reinterpret cast from i32 to u32 which will map
- // `i32::MIN..0` to `(i32::MAX as u32)..u32::MAX`
- ArrowUInt32Type::DATA_TYPE
- }
- _ => ArrowInt32Type::DATA_TYPE,
- }
- }
- PhysicalType::INT64 => {
- match target_type {
- ArrowType::UInt64 => {
- // follow C++ implementation and use overflow/reinterpret cast from i64 to u64 which will map
- // `i64::MIN..0` to `(i64::MAX as u64)..u64::MAX`
- ArrowUInt64Type::DATA_TYPE
- }
- _ => ArrowInt64Type::DATA_TYPE,
- }
- }
- PhysicalType::FLOAT => ArrowFloat32Type::DATA_TYPE,
- PhysicalType::DOUBLE => ArrowFloat64Type::DATA_TYPE,
- PhysicalType::INT96
- | PhysicalType::BYTE_ARRAY
- | PhysicalType::FIXED_LEN_BYTE_ARRAY => {
- unreachable!(
- "PrimitiveArrayReaders don't support complex physical types"
- );
- }
- };
-
- // Convert to arrays by using the Parquet phyisical type.
- // The physical types are then cast to Arrow types if necessary
-
- let mut record_data = self.record_reader.consume_record_data()?;
-
- if T::get_physical_type() == PhysicalType::BOOLEAN {
- let mut boolean_buffer = BooleanBufferBuilder::new(record_data.len());
-
- for e in record_data.as_slice() {
- boolean_buffer.append(*e > 0);
- }
- record_data = boolean_buffer.finish();
- }
-
- let mut array_data = ArrayDataBuilder::new(arrow_data_type)
- .len(self.record_reader.num_values())
- .add_buffer(record_data);
-
- if let Some(b) = self.record_reader.consume_bitmap_buffer()? {
- array_data = array_data.null_bit_buffer(b);
- }
-
- let array = match T::get_physical_type() {
- PhysicalType::BOOLEAN => {
- Arc::new(BooleanArray::from(array_data.build())) as ArrayRef
- }
- PhysicalType::INT32 => {
- Arc::new(PrimitiveArray::<ArrowInt32Type>::from(array_data.build()))
- as ArrayRef
- }
- PhysicalType::INT64 => {
- Arc::new(PrimitiveArray::<ArrowInt64Type>::from(array_data.build()))
- as ArrayRef
- }
- PhysicalType::FLOAT => {
- Arc::new(PrimitiveArray::<ArrowFloat32Type>::from(array_data.build()))
- as ArrayRef
- }
- PhysicalType::DOUBLE => {
- Arc::new(PrimitiveArray::<ArrowFloat64Type>::from(array_data.build()))
- as ArrayRef
- }
- PhysicalType::INT96
- | PhysicalType::BYTE_ARRAY
- | PhysicalType::FIXED_LEN_BYTE_ARRAY => {
- unreachable!(
- "PrimitiveArrayReaders don't support complex physical types"
- );
- }
- };
-
- // cast to Arrow type
- // We make a strong assumption here that the casts should be infallible.
- // If the cast fails because of incompatible datatypes, then there might
- // be a bigger problem with how Arrow schemas are converted to Parquet.
- //
- // As there is not always a 1:1 mapping between Arrow and Parquet, there
- // are datatypes which we must convert explicitly.
- // These are:
- // - date64: we should cast int32 to date32, then date32 to date64.
- let array = match target_type {
- ArrowType::Date64 => {
- // this is cheap as it internally reinterprets the data
- let a = arrow::compute::cast(&array, &ArrowType::Date32)?;
- arrow::compute::cast(&a, &target_type)?
- }
- ArrowType::Decimal(p, s) => {
- let mut builder = DecimalBuilder::new(array.len(), p, s);
- match array.data_type() {
- ArrowType::Int32 => {
- let values = array.as_any().downcast_ref::<Int32Array>().unwrap();
- for maybe_value in values.iter() {
- match maybe_value {
- Some(value) => builder.append_value(value as i128)?,
- None => builder.append_null()?,
- }
- }
- }
- ArrowType::Int64 => {
- let values = array.as_any().downcast_ref::<Int64Array>().unwrap();
- for maybe_value in values.iter() {
- match maybe_value {
- Some(value) => builder.append_value(value as i128)?,
- None => builder.append_null()?,
- }
- }
- }
- _ => {
- return Err(ArrowError(format!(
- "Cannot convert {:?} to decimal",
- array.data_type()
- )))
- }
- }
- Arc::new(builder.finish()) as ArrayRef
- }
- _ => arrow::compute::cast(&array, &target_type)?,
- };
-
- // save definition and repetition buffers
- self.def_levels_buffer = self.record_reader.consume_def_levels()?;
- self.rep_levels_buffer = self.record_reader.consume_rep_levels()?;
- self.record_reader.reset();
- Ok(array)
- }
-
- fn get_def_levels(&self) -> Option<&[i16]> {
- self.def_levels_buffer
- .as_ref()
- .map(|buf| unsafe { buf.typed_data() })
- }
-
- fn get_rep_levels(&self) -> Option<&[i16]> {
- self.rep_levels_buffer
- .as_ref()
- .map(|buf| unsafe { buf.typed_data() })
- }
-}
-
-/// Primitive array readers are leaves of array reader tree. They accept page iterator
-/// and read them into primitive arrays.
-pub struct ComplexObjectArrayReader<T, C>
-where
- T: DataType,
- C: Converter<Vec<Option<T::T>>, ArrayRef> + 'static,
-{
- data_type: ArrowType,
- pages: Box<dyn PageIterator>,
- def_levels_buffer: Option<Vec<i16>>,
- rep_levels_buffer: Option<Vec<i16>>,
- column_desc: ColumnDescPtr,
- column_reader: Option<ColumnReaderImpl<T>>,
- converter: C,
- _parquet_type_marker: PhantomData<T>,
- _converter_marker: PhantomData<C>,
-}
-
-impl<T, C> ArrayReader for ComplexObjectArrayReader<T, C>
-where
- T: DataType,
- C: Converter<Vec<Option<T::T>>, ArrayRef> + 'static,
-{
- fn as_any(&self) -> &dyn Any {
- self
- }
-
- fn get_data_type(&self) -> &ArrowType {
- &self.data_type
- }
-
- fn next_batch(&mut self, batch_size: usize) -> Result<ArrayRef> {
- // Try to initialize column reader
- if self.column_reader.is_none() {
- self.next_column_reader()?;
- }
-
- let mut data_buffer: Vec<T::T> = Vec::with_capacity(batch_size);
- data_buffer.resize_with(batch_size, T::T::default);
-
- let mut def_levels_buffer = if self.column_desc.max_def_level() > 0 {
- let mut buf: Vec<i16> = Vec::with_capacity(batch_size);
- buf.resize_with(batch_size, || 0);
- Some(buf)
- } else {
- None
- };
-
- let mut rep_levels_buffer = if self.column_desc.max_rep_level() > 0 {
- let mut buf: Vec<i16> = Vec::with_capacity(batch_size);
- buf.resize_with(batch_size, || 0);
- Some(buf)
- } else {
- None
- };
-
- let mut num_read = 0;
-
- while self.column_reader.is_some() && num_read < batch_size {
- let num_to_read = batch_size - num_read;
- let cur_data_buf = &mut data_buffer[num_read..];
- let cur_def_levels_buf =
- def_levels_buffer.as_mut().map(|b| &mut b[num_read..]);
- let cur_rep_levels_buf =
- rep_levels_buffer.as_mut().map(|b| &mut b[num_read..]);
- let (data_read, levels_read) =
- self.column_reader.as_mut().unwrap().read_batch(
- num_to_read,
- cur_def_levels_buf,
- cur_rep_levels_buf,
- cur_data_buf,
- )?;
-
- // Fill space
- if levels_read > data_read {
- def_levels_buffer.iter().for_each(|def_levels_buffer| {
- let (mut level_pos, mut data_pos) = (levels_read, data_read);
- while level_pos > 0 && data_pos > 0 {
- if def_levels_buffer[num_read + level_pos - 1]
- == self.column_desc.max_def_level()
- {
- cur_data_buf.swap(level_pos - 1, data_pos - 1);
- level_pos -= 1;
- data_pos -= 1;
- } else {
- level_pos -= 1;
- }
- }
- });
- }
-
- let values_read = max(levels_read, data_read);
- num_read += values_read;
- // current page exhausted && page iterator exhausted
- if values_read < num_to_read && !self.next_column_reader()? {
- break;
- }
- }
-
- data_buffer.truncate(num_read);
- def_levels_buffer
- .iter_mut()
- .for_each(|buf| buf.truncate(num_read));
- rep_levels_buffer
- .iter_mut()
- .for_each(|buf| buf.truncate(num_read));
-
- self.def_levels_buffer = def_levels_buffer;
- self.rep_levels_buffer = rep_levels_buffer;
-
- let data: Vec<Option<T::T>> = if self.def_levels_buffer.is_some() {
- data_buffer
- .into_iter()
- .zip(self.def_levels_buffer.as_ref().unwrap().iter())
- .map(|(t, def_level)| {
- if *def_level == self.column_desc.max_def_level() {
- Some(t)
- } else {
- None
- }
- })
- .collect()
- } else {
- data_buffer.into_iter().map(Some).collect()
- };
-
- let mut array = self.converter.convert(data)?;
-
- if let ArrowType::Dictionary(_, _) = self.data_type {
- array = arrow::compute::cast(&array, &self.data_type)?;
- }
-
- Ok(array)
- }
-
- fn get_def_levels(&self) -> Option<&[i16]> {
- self.def_levels_buffer.as_deref()
- }
-
- fn get_rep_levels(&self) -> Option<&[i16]> {
- self.rep_levels_buffer.as_deref()
- }
-}
-
-impl<T, C> ComplexObjectArrayReader<T, C>
-where
- T: DataType,
- C: Converter<Vec<Option<T::T>>, ArrayRef> + 'static,
-{
- pub fn new(
- pages: Box<dyn PageIterator>,
- column_desc: ColumnDescPtr,
- converter: C,
- arrow_type: Option<ArrowType>,
- ) -> Result<Self> {
- let data_type = match arrow_type {
- Some(t) => t,
- None => parquet_to_arrow_field(column_desc.as_ref())?
- .data_type()
- .clone(),
- };
-
- Ok(Self {
- data_type,
- pages,
- def_levels_buffer: None,
- rep_levels_buffer: None,
- column_desc,
- column_reader: None,
- converter,
- _parquet_type_marker: PhantomData,
- _converter_marker: PhantomData,
- })
- }
-
- fn next_column_reader(&mut self) -> Result<bool> {
- Ok(match self.pages.next() {
- Some(page) => {
- self.column_reader =
- Some(ColumnReaderImpl::<T>::new(self.column_desc.clone(), page?));
- true
- }
- None => false,
- })
- }
-}
-
-/// Implementation of list array reader.
-pub struct ListArrayReader<OffsetSize: OffsetSizeTrait> {
- item_reader: Box<dyn ArrayReader>,
- data_type: ArrowType,
- item_type: ArrowType,
- list_def_level: i16,
- list_rep_level: i16,
- list_empty_def_level: i16,
- list_null_def_level: i16,
- def_level_buffer: Option<Buffer>,
- rep_level_buffer: Option<Buffer>,
- _marker: PhantomData<OffsetSize>,
-}
-
-impl<OffsetSize: OffsetSizeTrait> ListArrayReader<OffsetSize> {
- /// Construct list array reader.
- pub fn new(
- item_reader: Box<dyn ArrayReader>,
- data_type: ArrowType,
- item_type: ArrowType,
- def_level: i16,
- rep_level: i16,
- list_null_def_level: i16,
- list_empty_def_level: i16,
- ) -> Self {
- Self {
- item_reader,
- data_type,
- item_type,
- list_def_level: def_level,
- list_rep_level: rep_level,
- list_null_def_level,
- list_empty_def_level,
- def_level_buffer: None,
- rep_level_buffer: None,
- _marker: PhantomData,
- }
- }
-}
-
-macro_rules! remove_primitive_array_indices {
- ($arr: expr, $item_type:ty, $indices:expr) => {{
- let array_data = match $arr.as_any().downcast_ref::<PrimitiveArray<$item_type>>() {
- Some(a) => a,
- _ => return Err(ParquetError::General(format!("Error generating next batch for ListArray: {:?} cannot be downcast to PrimitiveArray", $arr))),
- };
- let mut builder = PrimitiveBuilder::<$item_type>::new($arr.len());
- for i in 0..array_data.len() {
- if !$indices.contains(&i) {
- if array_data.is_null(i) {
- builder.append_null()?;
- } else {
- builder.append_value(array_data.value(i))?;
- }
- }
- }
- Ok(Arc::new(builder.finish()))
- }};
-}
-
-macro_rules! remove_array_indices_custom_builder {
- ($arr: expr, $array_type:ty, $item_builder:ident, $indices:expr) => {{
- let array_data = match $arr.as_any().downcast_ref::<$array_type>() {
- Some(a) => a,
- _ => return Err(ParquetError::General(format!("Error generating next batch for ListArray: {:?} cannot be downcast to PrimitiveArray", $arr))),
- };
- let mut builder = $item_builder::new(array_data.len());
-
- for i in 0..array_data.len() {
- if !$indices.contains(&i) {
- if array_data.is_null(i) {
- builder.append_null()?;
- } else {
- builder.append_value(array_data.value(i))?;
- }
- }
- }
- Ok(Arc::new(builder.finish()))
- }};
-}
-
-macro_rules! remove_fixed_size_binary_array_indices {
- ($arr: expr, $array_type:ty, $item_builder:ident, $indices:expr, $len:expr) => {{
- let array_data = match $arr.as_any().downcast_ref::<$array_type>() {
- Some(a) => a,
- _ => return Err(ParquetError::General(format!("Error generating next batch for ListArray: {:?} cannot be downcast to PrimitiveArray", $arr))),
- };
- let mut builder = FixedSizeBinaryBuilder::new(array_data.len(), $len);
- for i in 0..array_data.len() {
- if !$indices.contains(&i) {
- if array_data.is_null(i) {
- builder.append_null()?;
- } else {
- builder.append_value(array_data.value(i))?;
- }
- }
- }
- Ok(Arc::new(builder.finish()))
- }};
-}
-
-fn remove_indices(
- arr: ArrayRef,
- item_type: ArrowType,
- indices: Vec<usize>,
-) -> Result<ArrayRef> {
- match item_type {
- ArrowType::UInt8 => remove_primitive_array_indices!(arr, ArrowUInt8Type, indices),
- ArrowType::UInt16 => {
- remove_primitive_array_indices!(arr, ArrowUInt16Type, indices)
- }
- ArrowType::UInt32 => {
- remove_primitive_array_indices!(arr, ArrowUInt32Type, indices)
- }
- ArrowType::UInt64 => {
- remove_primitive_array_indices!(arr, ArrowUInt64Type, indices)
- }
- ArrowType::Int8 => remove_primitive_array_indices!(arr, ArrowInt8Type, indices),
- ArrowType::Int16 => remove_primitive_array_indices!(arr, ArrowInt16Type, indices),
- ArrowType::Int32 => remove_primitive_array_indices!(arr, ArrowInt32Type, indices),
- ArrowType::Int64 => remove_primitive_array_indices!(arr, ArrowInt64Type, indices),
- ArrowType::Float32 => {
- remove_primitive_array_indices!(arr, ArrowFloat32Type, indices)
- }
- ArrowType::Float64 => {
- remove_primitive_array_indices!(arr, ArrowFloat64Type, indices)
- }
- ArrowType::Boolean => {
- remove_array_indices_custom_builder!(
- arr,
- BooleanArray,
- BooleanBuilder,
- indices
- )
- }
- ArrowType::Date32 => {
- remove_primitive_array_indices!(arr, ArrowDate32Type, indices)
- }
- ArrowType::Date64 => {
- remove_primitive_array_indices!(arr, ArrowDate64Type, indices)
- }
- ArrowType::Time32(ArrowTimeUnit::Second) => {
- remove_primitive_array_indices!(arr, ArrowTime32SecondType, indices)
- }
- ArrowType::Time32(ArrowTimeUnit::Millisecond) => {
- remove_primitive_array_indices!(arr, ArrowTime32MillisecondType, indices)
- }
- ArrowType::Time64(ArrowTimeUnit::Microsecond) => {
- remove_primitive_array_indices!(arr, ArrowTime64MicrosecondType, indices)
- }
- ArrowType::Time64(ArrowTimeUnit::Nanosecond) => {
- remove_primitive_array_indices!(arr, ArrowTime64NanosecondType, indices)
- }
- ArrowType::Duration(ArrowTimeUnit::Second) => {
- remove_primitive_array_indices!(arr, ArrowDurationSecondType, indices)
- }
- ArrowType::Duration(ArrowTimeUnit::Millisecond) => {
- remove_primitive_array_indices!(arr, ArrowDurationMillisecondType, indices)
- }
- ArrowType::Duration(ArrowTimeUnit::Microsecond) => {
- remove_primitive_array_indices!(arr, ArrowDurationMicrosecondType, indices)
- }
- ArrowType::Duration(ArrowTimeUnit::Nanosecond) => {
- remove_primitive_array_indices!(arr, ArrowDurationNanosecondType, indices)
- }
- ArrowType::Timestamp(ArrowTimeUnit::Second, _) => {
- remove_primitive_array_indices!(arr, ArrowTimestampSecondType, indices)
- }
- ArrowType::Timestamp(ArrowTimeUnit::Millisecond, _) => {
- remove_primitive_array_indices!(arr, ArrowTimestampMillisecondType, indices)
- }
- ArrowType::Timestamp(ArrowTimeUnit::Microsecond, _) => {
- remove_primitive_array_indices!(arr, ArrowTimestampMicrosecondType, indices)
- }
- ArrowType::Timestamp(ArrowTimeUnit::Nanosecond, _) => {
- remove_primitive_array_indices!(arr, ArrowTimestampNanosecondType, indices)
- }
- ArrowType::Utf8 => {
- remove_array_indices_custom_builder!(arr, StringArray, StringBuilder, indices)
- }
- ArrowType::Binary => {
- remove_array_indices_custom_builder!(arr, BinaryArray, BinaryBuilder, indices)
- }
- ArrowType::FixedSizeBinary(size) => remove_fixed_size_binary_array_indices!(
- arr,
- FixedSizeBinaryArray,
- FixedSizeBinaryBuilder,
- indices,
- size
- ),
- _ => Err(ParquetError::General(format!(
- "ListArray of type List({:?}) is not supported by array_reader",
- item_type
- ))),
- }
-}
-
-/// Implementation of ListArrayReader. Nested lists and lists of structs are not yet supported.
-impl<OffsetSize: OffsetSizeTrait> ArrayReader for ListArrayReader<OffsetSize> {
- fn as_any(&self) -> &dyn Any {
- self
- }
-
- /// Returns data type.
- /// This must be a List.
- fn get_data_type(&self) -> &ArrowType {
- &self.data_type
- }
-
- fn next_batch(&mut self, batch_size: usize) -> Result<ArrayRef> {
- let next_batch_array = self.item_reader.next_batch(batch_size)?;
- let item_type = self.item_reader.get_data_type().clone();
-
- if next_batch_array.len() == 0 {
- return Ok(new_empty_array(&self.data_type));
- }
- let def_levels = self
- .item_reader
- .get_def_levels()
- .ok_or_else(|| ArrowError("item_reader def levels are None.".to_string()))?;
- let rep_levels = self
- .item_reader
- .get_rep_levels()
- .ok_or_else(|| ArrowError("item_reader rep levels are None.".to_string()))?;
-
- if !((def_levels.len() == rep_levels.len())
- && (rep_levels.len() == next_batch_array.len()))
- {
- return Err(ArrowError(
- "Expected item_reader def_levels and rep_levels to be same length as batch".to_string(),
- ));
- }
-
- // List definitions can be encoded as 4 values:
- // - n + 0: the list slot is null
- // - n + 1: the list slot is not null, but is empty (i.e. [])
- // - n + 2: the list slot is not null, but its child is empty (i.e. [ null ])
- // - n + 3: the list slot is not null, and its child is not empty
- // Where n is the max definition level of the list's parent.
- // If a Parquet schema's only leaf is the list, then n = 0.
-
- // If the list index is at empty definition, the child slot is null
- let null_list_indices: Vec<usize> = def_levels
- .iter()
- .enumerate()
- .filter_map(|(index, def)| {
- if *def <= self.list_empty_def_level {
- Some(index)
- } else {
- None
- }
- })
- .collect();
- let batch_values = match null_list_indices.len() {
- 0 => next_batch_array.clone(),
- _ => remove_indices(next_batch_array.clone(), item_type, null_list_indices)?,
- };
-
- // first item in each list has rep_level = 0, subsequent items have rep_level = 1
- let mut offsets: Vec<OffsetSize> = Vec::new();
- let mut cur_offset = OffsetSize::zero();
- def_levels.iter().zip(rep_levels).for_each(|(d, r)| {
- if *r == 0 || d == &self.list_empty_def_level {
- offsets.push(cur_offset);
- }
- if d > &self.list_empty_def_level {
- cur_offset += OffsetSize::one();
- }
- });
- offsets.push(cur_offset);
-
- let num_bytes = bit_util::ceil(offsets.len(), 8);
- // TODO: A useful optimization is to use the null count to fill with
- // 0 or null, to reduce individual bits set in a loop.
- // To favour dense data, set every slot to true, then unset
- let mut null_buf = MutableBuffer::new(num_bytes).with_bitset(num_bytes, true);
- let null_slice = null_buf.as_slice_mut();
- let mut list_index = 0;
- for i in 0..rep_levels.len() {
- // If the level is lower than empty, then the slot is null.
- // When a list is non-nullable, its empty level = null level,
- // so this automatically factors that in.
- if rep_levels[i] == 0 && def_levels[i] < self.list_empty_def_level {
- bit_util::unset_bit(null_slice, list_index);
- }
- if rep_levels[i] == 0 {
- list_index += 1;
- }
- }
- let value_offsets = Buffer::from(&offsets.to_byte_slice());
-
- let list_data = ArrayData::builder(self.get_data_type().clone())
- .len(offsets.len() - 1)
- .add_buffer(value_offsets)
- .add_child_data(batch_values.data().clone())
- .null_bit_buffer(null_buf.into())
- .offset(next_batch_array.offset())
- .build();
-
- let result_array = GenericListArray::<OffsetSize>::from(list_data);
- Ok(Arc::new(result_array))
- }
-
- fn get_def_levels(&self) -> Option<&[i16]> {
- self.def_level_buffer
- .as_ref()
- .map(|buf| unsafe { buf.typed_data() })
- }
-
- fn get_rep_levels(&self) -> Option<&[i16]> {
- self.rep_level_buffer
- .as_ref()
- .map(|buf| unsafe { buf.typed_data() })
- }
-}
-
-/// Implementation of struct array reader.
-pub struct StructArrayReader {
- children: Vec<Box<dyn ArrayReader>>,
- data_type: ArrowType,
- struct_def_level: i16,
- struct_rep_level: i16,
- def_level_buffer: Option<Buffer>,
- rep_level_buffer: Option<Buffer>,
-}
-
-impl StructArrayReader {
- /// Construct struct array reader.
- pub fn new(
- data_type: ArrowType,
- children: Vec<Box<dyn ArrayReader>>,
- def_level: i16,
- rep_level: i16,
- ) -> Self {
- Self {
- data_type,
- children,
- struct_def_level: def_level,
- struct_rep_level: rep_level,
- def_level_buffer: None,
- rep_level_buffer: None,
- }
- }
-}
-
-impl ArrayReader for StructArrayReader {
- fn as_any(&self) -> &dyn Any {
- self
- }
-
- /// Returns data type.
- /// This must be a struct.
- fn get_data_type(&self) -> &ArrowType {
- &self.data_type
- }
-
- /// Read `batch_size` struct records.
- ///
- /// Definition levels of struct array is calculated as following:
- /// ```ignore
- /// def_levels[i] = min(child1_def_levels[i], child2_def_levels[i], ...,
- /// childn_def_levels[i]);
- /// ```
- ///
- /// Repetition levels of struct array is calculated as following:
- /// ```ignore
- /// rep_levels[i] = child1_rep_levels[i];
- /// ```
- ///
- /// The null bitmap of struct array is calculated from def_levels:
- /// ```ignore
- /// null_bitmap[i] = (def_levels[i] >= self.def_level);
- /// ```
- fn next_batch(&mut self, batch_size: usize) -> Result<ArrayRef> {
- if self.children.is_empty() {
- self.def_level_buffer = None;
- self.rep_level_buffer = None;
- return Ok(Arc::new(StructArray::from(Vec::new())));
- }
-
- let children_array = self
- .children
- .iter_mut()
- .map(|reader| reader.next_batch(batch_size))
- .try_fold(
- Vec::new(),
- |mut result, child_array| -> Result<Vec<ArrayRef>> {
- result.push(child_array?);
- Ok(result)
- },
- )?;
-
- // check that array child data has same size
- let children_array_len =
- children_array.first().map(|arr| arr.len()).ok_or_else(|| {
- general_err!("Struct array reader should have at least one child!")
- })?;
-
- let all_children_len_eq = children_array
- .iter()
- .all(|arr| arr.len() == children_array_len);
- if !all_children_len_eq {
- return Err(general_err!("Not all children array length are the same!"));
- }
-
- // calculate struct def level data
- let buffer_size = children_array_len * size_of::<i16>();
- let mut def_level_data_buffer = MutableBuffer::new(buffer_size);
- def_level_data_buffer.resize(buffer_size, 0);
-
- let def_level_data = def_level_data_buffer.typed_data_mut();
-
- def_level_data
- .iter_mut()
- .for_each(|v| *v = self.struct_def_level);
-
- for child in &self.children {
- if let Some(current_child_def_levels) = child.get_def_levels() {
- if current_child_def_levels.len() != children_array_len {
- return Err(general_err!("Child array length are not equal!"));
- } else {
- for i in 0..children_array_len {
- def_level_data[i] =
- min(def_level_data[i], current_child_def_levels[i]);
- }
- }
- }
- }
-
- // calculate bitmap for current array
- let mut bitmap_builder = BooleanBufferBuilder::new(children_array_len);
- for def_level in def_level_data {
- let not_null = *def_level >= self.struct_def_level;
- bitmap_builder.append(not_null);
- }
-
- // Now we can build array data
- let array_data = ArrayDataBuilder::new(self.data_type.clone())
- .len(children_array_len)
- .null_bit_buffer(bitmap_builder.finish())
- .child_data(
- children_array
- .iter()
- .map(|x| x.data().clone())
- .collect::<Vec<ArrayData>>(),
- )
- .build();
-
- // calculate struct rep level data, since struct doesn't add to repetition
- // levels, here we just need to keep repetition levels of first array
- // TODO: Verify that all children array reader has same repetition levels
- let rep_level_data = self
- .children
- .first()
- .ok_or_else(|| {
- general_err!("Struct array reader should have at least one child!")
- })?
- .get_rep_levels()
- .map(|data| -> Result<Buffer> {
- let mut buffer = Int16BufferBuilder::new(children_array_len);
- buffer.append_slice(data);
- Ok(buffer.finish())
- })
- .transpose()?;
-
- self.def_level_buffer = Some(def_level_data_buffer.into());
- self.rep_level_buffer = rep_level_data;
- Ok(Arc::new(StructArray::from(array_data)))
- }
-
- fn get_def_levels(&self) -> Option<&[i16]> {
- self.def_level_buffer
- .as_ref()
- .map(|buf| unsafe { buf.typed_data() })
- }
-
- fn get_rep_levels(&self) -> Option<&[i16]> {
- self.rep_level_buffer
- .as_ref()
- .map(|buf| unsafe { buf.typed_data() })
- }
-}
-
-/// Create array reader from parquet schema, column indices, and parquet file reader.
-pub fn build_array_reader<T>(
- parquet_schema: SchemaDescPtr,
- arrow_schema: Schema,
- column_indices: T,
- file_reader: Arc<dyn FileReader>,
-) -> Result<Box<dyn ArrayReader>>
-where
- T: IntoIterator<Item = usize>,
-{
- let mut leaves = HashMap::<*const Type, usize>::new();
-
- let mut filtered_root_names = HashSet::<String>::new();
-
- for c in column_indices {
- let column = parquet_schema.column(c).self_type() as *const Type;
-
- leaves.insert(column, c);
-
- let root = parquet_schema.get_column_root_ptr(c);
- filtered_root_names.insert(root.name().to_string());
- }
-
- if leaves.is_empty() {
- return Err(general_err!("Can't build array reader without columns!"));
- }
-
- // Only pass root fields that take part in the projection
- // to avoid traversal of columns that are not read.
- // TODO: also prune unread parts of the tree in child structures
- let filtered_root_fields = parquet_schema
- .root_schema()
- .get_fields()
- .iter()
- .filter(|field| filtered_root_names.contains(field.name()))
- .cloned()
- .collect::<Vec<_>>();
-
- let proj = Type::GroupType {
- basic_info: parquet_schema.root_schema().get_basic_info().clone(),
- fields: filtered_root_fields,
- };
-
- ArrayReaderBuilder::new(
- Arc::new(proj),
- Arc::new(arrow_schema),
- Arc::new(leaves),
- file_reader,
- )
- .build_array_reader()
-}
-
-/// Used to build array reader.
-struct ArrayReaderBuilder {
- root_schema: TypePtr,
- arrow_schema: Arc<Schema>,
- // Key: columns that need to be included in final array builder
- // Value: column index in schema
- columns_included: Arc<HashMap<*const Type, usize>>,
- file_reader: Arc<dyn FileReader>,
-}
-
-/// Used in type visitor.
-#[derive(Clone)]
-struct ArrayReaderBuilderContext {
- def_level: i16,
- rep_level: i16,
- path: ColumnPath,
-}
-
-impl Default for ArrayReaderBuilderContext {
- fn default() -> Self {
- Self {
- def_level: 0i16,
- rep_level: 0i16,
- path: ColumnPath::new(Vec::new()),
- }
- }
-}
-
-/// Create array reader by visiting schema.
-impl<'a> TypeVisitor<Option<Box<dyn ArrayReader>>, &'a ArrayReaderBuilderContext>
- for ArrayReaderBuilder
-{
- /// Build array reader for primitive type.
- /// Currently we don't have a list reader implementation, so repeated type is not
- /// supported yet.
- fn visit_primitive(
- &mut self,
- cur_type: TypePtr,
- context: &'a ArrayReaderBuilderContext,
- ) -> Result<Option<Box<dyn ArrayReader>>> {
- if self.is_included(cur_type.as_ref()) {
- let mut new_context = context.clone();
- new_context.path.append(vec![cur_type.name().to_string()]);
-
- match cur_type.get_basic_info().repetition() {
- Repetition::REPEATED => {
- new_context.def_level += 1;
- new_context.rep_level += 1;
- }
- Repetition::OPTIONAL => {
- new_context.def_level += 1;
- }
- _ => (),
- }
-
- let reader =
- self.build_for_primitive_type_inner(cur_type.clone(), &new_context)?;
-
- if cur_type.get_basic_info().repetition() == Repetition::REPEATED {
- Err(ArrowError(
- "Reading repeated field is not supported yet!".to_string(),
- ))
- } else {
- Ok(Some(reader))
- }
- } else {
- Ok(None)
- }
- }
-
- /// Build array reader for struct type.
- fn visit_struct(
- &mut self,
- cur_type: Arc<Type>,
- context: &'a ArrayReaderBuilderContext,
- ) -> Result<Option<Box<dyn ArrayReader>>> {
- let mut new_context = context.clone();
- new_context.path.append(vec![cur_type.name().to_string()]);
-
- if cur_type.get_basic_info().has_repetition() {
- match cur_type.get_basic_info().repetition() {
- Repetition::REPEATED => {
- new_context.def_level += 1;
- new_context.rep_level += 1;
- }
- Repetition::OPTIONAL => {
- new_context.def_level += 1;
- }
- _ => (),
- }
- }
-
- if let Some(reader) = self.build_for_struct_type_inner(&cur_type, &new_context)? {
- if cur_type.get_basic_info().has_repetition()
- && cur_type.get_basic_info().repetition() == Repetition::REPEATED
- {
- Err(ArrowError(
- "Reading repeated field is not supported yet!".to_string(),
- ))
- } else {
- Ok(Some(reader))
- }
- } else {
- Ok(None)
- }
- }
-
- /// Build array reader for map type.
- /// Currently this is not supported.
- fn visit_map(
- &mut self,
- _cur_type: Arc<Type>,
- _context: &'a ArrayReaderBuilderContext,
- ) -> Result<Option<Box<dyn ArrayReader>>> {
- Err(ArrowError(
- "Reading parquet map array into arrow is not supported yet!".to_string(),
- ))
- }
-
- /// Build array reader for list type.
- fn visit_list_with_item(
- &mut self,
- list_type: Arc<Type>,
- item_type: Arc<Type>,
- context: &'a ArrayReaderBuilderContext,
- ) -> Result<Option<Box<dyn ArrayReader>>> {
- let list_child = &list_type
- .get_fields()
- .first()
- .ok_or_else(|| ArrowError("List field must have a child.".to_string()))?;
- let mut new_context = context.clone();
-
- new_context.path.append(vec![list_type.name().to_string()]);
- // We need to know at what definition a list or its child is null
- let list_null_def = new_context.def_level;
- let mut list_empty_def = new_context.def_level;
-
- // If the list's root is nullable
- if let Repetition::OPTIONAL = list_type.get_basic_info().repetition() {
- new_context.def_level += 1;
- // current level is nullable, increment to get level for empty list slot
- list_empty_def += 1;
- }
-
- match list_child.get_basic_info().repetition() {
- Repetition::REPEATED => {
- new_context.def_level += 1;
- new_context.rep_level += 1;
- }
- Repetition::OPTIONAL => {
- new_context.def_level += 1;
- }
- _ => (),
- }
-
- let item_reader = self
- .dispatch(item_type.clone(), &new_context)
- .unwrap()
- .unwrap();
-
- let item_reader_type = item_reader.get_data_type().clone();
-
- match item_reader_type {
- ArrowType::List(_)
- | ArrowType::FixedSizeList(_, _)
- | ArrowType::Struct(_)
- | ArrowType::Dictionary(_, _) => Err(ArrowError(format!(
- "reading List({:?}) into arrow not supported yet",
- item_type
- ))),
- _ => {
- // a list is a group type with a single child. The list child's
- // name comes from the child's field name.
- let mut list_child = list_type.get_fields().first().ok_or(ArrowError(
- "List GroupType should have a field".to_string(),
- ))?;
- // if the child's name is "list" and it has a child, then use this child
- if list_child.name() == "list" && !list_child.get_fields().is_empty() {
- list_child = list_child.get_fields().first().unwrap();
- }
- let arrow_type = self
- .arrow_schema
- .field_with_name(list_type.name())
- .ok()
- .map(|f| f.data_type().to_owned())
- .unwrap_or_else(|| {
- ArrowType::List(Box::new(Field::new(
- list_child.name(),
- item_reader_type.clone(),
- list_child.is_optional(),
- )))
- });
-
- let list_array_reader: Box<dyn ArrayReader> = match arrow_type {
- ArrowType::List(_) => Box::new(ListArrayReader::<i32>::new(
- item_reader,
- arrow_type,
- item_reader_type,
- new_context.def_level,
- new_context.rep_level,
- list_null_def,
- list_empty_def,
- )),
- ArrowType::LargeList(_) => Box::new(ListArrayReader::<i64>::new(
- item_reader,
- arrow_type,
- item_reader_type,
- new_context.def_level,
- new_context.rep_level,
- list_null_def,
- list_empty_def,
- )),
-
- _ => {
- return Err(ArrowError(format!(
- "creating ListArrayReader with type {:?} should be unreachable",
- arrow_type
- )))
- }
- };
-
- Ok(Some(list_array_reader))
- }
- }
- }
-}
-
-impl<'a> ArrayReaderBuilder {
- /// Construct array reader builder.
- fn new(
- root_schema: TypePtr,
- arrow_schema: Arc<Schema>,
- columns_included: Arc<HashMap<*const Type, usize>>,
- file_reader: Arc<dyn FileReader>,
- ) -> Self {
- Self {
- root_schema,
- arrow_schema,
- columns_included,
- file_reader,
- }
- }
-
- /// Main entry point.
- fn build_array_reader(&mut self) -> Result<Box<dyn ArrayReader>> {
- let context = ArrayReaderBuilderContext::default();
-
- self.visit_struct(self.root_schema.clone(), &context)
- .and_then(|reader_opt| {
- reader_opt.ok_or_else(|| general_err!("Failed to build array reader!"))
- })
- }
-
- // Utility functions
-
- /// Check whether one column in included in this array reader builder.
- fn is_included(&self, t: &Type) -> bool {
- self.columns_included.contains_key(&(t as *const Type))
- }
-
- /// Creates primitive array reader for each primitive type.
- fn build_for_primitive_type_inner(
- &self,
- cur_type: TypePtr,
- context: &'a ArrayReaderBuilderContext,
- ) -> Result<Box<dyn ArrayReader>> {
- let column_desc = Arc::new(ColumnDescriptor::new(
- cur_type.clone(),
- context.def_level,
- context.rep_level,
- context.path.clone(),
- ));
- let page_iterator = Box::new(FilePageIterator::new(
- self.columns_included[&(cur_type.as_ref() as *const Type)],
- self.file_reader.clone(),
- )?);
-
- let arrow_type: Option<ArrowType> = self
- .get_arrow_field(&cur_type, context)
- .map(|f| f.data_type().clone());
-
- match cur_type.get_physical_type() {
- PhysicalType::BOOLEAN => Ok(Box::new(PrimitiveArrayReader::<BoolType>::new(
- page_iterator,
- column_desc,
- arrow_type,
- )?)),
- PhysicalType::INT32 => {
- if let Some(ArrowType::Null) = arrow_type {
- Ok(Box::new(NullArrayReader::<Int32Type>::new(
- page_iterator,
- column_desc,
- )?))
- } else {
- Ok(Box::new(PrimitiveArrayReader::<Int32Type>::new(
- page_iterator,
- column_desc,
- arrow_type,
- )?))
- }
- }
- PhysicalType::INT64 => Ok(Box::new(PrimitiveArrayReader::<Int64Type>::new(
- page_iterator,
- column_desc,
- arrow_type,
- )?)),
- PhysicalType::INT96 => {
- // get the optional timezone information from arrow type
- let timezone = arrow_type
- .as_ref()
- .map(|data_type| {
- if let ArrowType::Timestamp(_, tz) = data_type {
- tz.clone()
- } else {
- None
- }
- })
- .flatten();
- let converter = Int96Converter::new(Int96ArrayConverter { timezone });
- Ok(Box::new(ComplexObjectArrayReader::<
- Int96Type,
- Int96Converter,
- >::new(
- page_iterator,
- column_desc,
- converter,
- arrow_type,
- )?))
- }
- PhysicalType::FLOAT => Ok(Box::new(PrimitiveArrayReader::<FloatType>::new(
- page_iterator,
- column_desc,
- arrow_type,
- )?)),
- PhysicalType::DOUBLE => {
- Ok(Box::new(PrimitiveArrayReader::<DoubleType>::new(
- page_iterator,
- column_desc,
- arrow_type,
- )?))
- }
- PhysicalType::BYTE_ARRAY => {
- if cur_type.get_basic_info().converted_type() == ConvertedType::UTF8 {
- if let Some(ArrowType::LargeUtf8) = arrow_type {
- let converter =
- LargeUtf8Converter::new(LargeUtf8ArrayConverter {});
- Ok(Box::new(ComplexObjectArrayReader::<
- ByteArrayType,
- LargeUtf8Converter,
- >::new(
- page_iterator,
- column_desc,
- converter,
- arrow_type,
- )?))
- } else {
- use crate::arrow::arrow_array_reader::{
- ArrowArrayReader, StringArrayConverter,
- };
- let converter = StringArrayConverter::new();
- Ok(Box::new(ArrowArrayReader::try_new(
- *page_iterator,
- column_desc,
- converter,
- arrow_type,
- )?))
- }
- } else if let Some(ArrowType::LargeBinary) = arrow_type {
- let converter =
- LargeBinaryConverter::new(LargeBinaryArrayConverter {});
- Ok(Box::new(ComplexObjectArrayReader::<
- ByteArrayType,
- LargeBinaryConverter,
- >::new(
- page_iterator,
- column_desc,
- converter,
- arrow_type,
- )?))
- } else {
- let converter = BinaryConverter::new(BinaryArrayConverter {});
- Ok(Box::new(ComplexObjectArrayReader::<
- ByteArrayType,
- BinaryConverter,
- >::new(
- page_iterator,
- column_desc,
- converter,
- arrow_type,
- )?))
- }
- }
- PhysicalType::FIXED_LEN_BYTE_ARRAY
- if cur_type.get_basic_info().converted_type()
- == ConvertedType::DECIMAL =>
- {
- let converter = DecimalConverter::new(DecimalArrayConverter::new(
- cur_type.get_precision(),
- cur_type.get_scale(),
- ));
- Ok(Box::new(ComplexObjectArrayReader::<
- FixedLenByteArrayType,
- DecimalConverter,
- >::new(
- page_iterator,
- column_desc,
- converter,
- arrow_type,
- )?))
- }
- PhysicalType::FIXED_LEN_BYTE_ARRAY => {
- if cur_type.get_basic_info().converted_type() == ConvertedType::INTERVAL {
- let byte_width = match *cur_type {
- Type::PrimitiveType {
- ref type_length, ..
- } => *type_length,
- _ => {
- return Err(ArrowError(
- "Expected a physical type, not a group type".to_string(),
- ))
- }
- };
- if byte_width != 12 {
- return Err(ArrowError(format!(
- "Parquet interval type should have length of 12, found {}",
- byte_width
- )));
- }
- match arrow_type {
- Some(ArrowType::Interval(IntervalUnit::DayTime)) => {
- let converter = IntervalDayTimeConverter::new(
- IntervalDayTimeArrayConverter {},
- );
- Ok(Box::new(ComplexObjectArrayReader::<
- FixedLenByteArrayType,
- IntervalDayTimeConverter,
- >::new(
- page_iterator,
- column_desc,
- converter,
- arrow_type,
- )?))
- }
- Some(ArrowType::Interval(IntervalUnit::YearMonth)) => {
- let converter = IntervalYearMonthConverter::new(
- IntervalYearMonthArrayConverter {},
- );
- Ok(Box::new(ComplexObjectArrayReader::<
- FixedLenByteArrayType,
- IntervalYearMonthConverter,
- >::new(
- page_iterator,
- column_desc,
- converter,
- arrow_type,
- )?))
- }
- Some(t) => Err(ArrowError(format!(
- "Cannot write a Parquet interval to {:?}",
- t
- ))),
- None => {
- // we do not support an interval not matched to an Arrow type,
- // because we risk data loss as we won't know which of the 12 bytes
- // are or should be populated
- Err(ArrowError(
- "Cannot write a Parquet interval with no Arrow type specified.
- There is a risk of data loss as Arrow either supports YearMonth or
- DayTime precision. Without the Arrow type, we cannot infer the type.
- ".to_string()
- ))
- }
- }
- } else {
- let byte_width = match *cur_type {
- Type::PrimitiveType {
- ref type_length, ..
- } => *type_length,
- _ => {
- return Err(ArrowError(
- "Expected a physical type, not a group type".to_string(),
- ))
- }
- };
- let converter = FixedLenBinaryConverter::new(
- FixedSizeArrayConverter::new(byte_width),
- );
- Ok(Box::new(ComplexObjectArrayReader::<
- FixedLenByteArrayType,
- FixedLenBinaryConverter,
- >::new(
- page_iterator,
- column_desc,
- converter,
- arrow_type,
- )?))
- }
- }
- }
- }
-
- /// Constructs struct array reader without considering repetition.
- fn build_for_struct_type_inner(
- &mut self,
- cur_type: &Type,
- context: &'a ArrayReaderBuilderContext,
- ) -> Result<Option<Box<dyn ArrayReader>>> {
- let mut fields = Vec::with_capacity(cur_type.get_fields().len());
- let mut children_reader = Vec::with_capacity(cur_type.get_fields().len());
-
- for child in cur_type.get_fields() {
- let mut struct_context = context.clone();
- if let Some(child_reader) = self.dispatch(child.clone(), context)? {
- // TODO: this results in calling get_arrow_field twice, it could be reused
- // from child_reader above, by making child_reader carry its `Field`
- struct_context.path.append(vec![child.name().to_string()]);
- let field = match self.get_arrow_field(child, &struct_context) {
- Some(f) => f.clone(),
- _ => Field::new(
- child.name(),
- child_reader.get_data_type().clone(),
- child.is_optional(),
- ),
- };
- fields.push(field);
- children_reader.push(child_reader);
- }
- }
-
- if !fields.is_empty() {
- let arrow_type = ArrowType::Struct(fields);
- Ok(Some(Box::new(StructArrayReader::new(
- arrow_type,
- children_reader,
- context.def_level,
- context.rep_level,
- ))))
- } else {
- Ok(None)
- }
- }
-
- fn get_arrow_field(
- &self,
- cur_type: &Type,
- context: &'a ArrayReaderBuilderContext,
- ) -> Option<&Field> {
- let parts: Vec<&str> = context
- .path
- .parts()
- .iter()
- .map(|x| -> &str { x })
- .collect::<Vec<&str>>();
-
- // If the parts length is one it'll have the top level "schema" type. If
- // it's two then it'll be a top-level type that we can get from the arrow
- // schema directly.
- if parts.len() <= 2 {
- self.arrow_schema.field_with_name(cur_type.name()).ok()
- } else {
- // If it's greater than two then we need to traverse the type path
- // until we find the actual field we're looking for.
- let mut field: Option<&Field> = None;
-
- for (i, part) in parts.iter().enumerate().skip(1) {
- if i == 1 {
- field = self.arrow_schema.field_with_name(part).ok();
- } else if let Some(f) = field {
- if let ArrowType::Struct(fields) = f.data_type() {
- field = fields.iter().find(|f| f.name() == part)
- } else {
- field = None
- }
- } else {
- field = None
- }
- }
- field
- }
- }
-}
-
-#[cfg(test)]
-mod tests {
- use super::*;
- use crate::arrow::converter::{Utf8ArrayConverter, Utf8Converter};
- use crate::arrow::schema::parquet_to_arrow_schema;
- use crate::basic::{Encoding, Type as PhysicalType};
- use crate::column::page::{Page, PageReader};
- use crate::data_type::{ByteArray, DataType, Int32Type, Int64Type};
- use crate::errors::Result;
- use crate::file::reader::{FileReader, SerializedFileReader};
- use crate::schema::parser::parse_message_type;
- use crate::schema::types::{ColumnDescPtr, SchemaDescriptor};
- use crate::util::test_common::page_util::{
- DataPageBuilder, DataPageBuilderImpl, InMemoryPageIterator,
- };
- use crate::util::test_common::{get_test_file, make_pages};
- use arrow::array::{
- Array, ArrayRef, LargeListArray, ListArray, PrimitiveArray, StringArray,
- StructArray,
- };
- use arrow::datatypes::{
- ArrowPrimitiveType, DataType as ArrowType, Date32Type as ArrowDate32, Field,
- Int32Type as ArrowInt32, Int64Type as ArrowInt64,
- Time32MillisecondType as ArrowTime32MillisecondArray,
- Time64MicrosecondType as ArrowTime64MicrosecondArray,
- TimestampMicrosecondType as ArrowTimestampMicrosecondType,
- TimestampMillisecondType as ArrowTimestampMillisecondType,
- };
- use rand::distributions::uniform::SampleUniform;
- use rand::{thread_rng, Rng};
- use std::any::Any;
- use std::collections::VecDeque;
- use std::sync::Arc;
-
- fn make_column_chunks<T: DataType>(
- column_desc: ColumnDescPtr,
- encoding: Encoding,
- num_levels: usize,
- min_value: T::T,
- max_value: T::T,
- def_levels: &mut Vec<i16>,
- rep_levels: &mut Vec<i16>,
- values: &mut Vec<T::T>,
- page_lists: &mut Vec<Vec<Page>>,
- use_v2: bool,
- num_chunks: usize,
- ) where
- T::T: PartialOrd + SampleUniform + Copy,
- {
- for _i in 0..num_chunks {
- let mut pages = VecDeque::new();
- let mut data = Vec::new();
- let mut page_def_levels = Vec::new();
- let mut page_rep_levels = Vec::new();
-
- make_pages::<T>(
- column_desc.clone(),
- encoding,
- 1,
- num_levels,
- min_value,
- max_value,
- &mut page_def_levels,
- &mut page_rep_levels,
- &mut data,
- &mut pages,
- use_v2,
- );
-
- def_levels.append(&mut page_def_levels);
- rep_levels.append(&mut page_rep_levels);
- values.append(&mut data);
- page_lists.push(Vec::from(pages));
- }
- }
-
- #[test]
- fn test_primitive_array_reader_empty_pages() {
- // Construct column schema
- let message_type = "
- message test_schema {
- REQUIRED INT32 leaf;
- }
- ";
-
- let schema = parse_message_type(message_type)
- .map(|t| Arc::new(SchemaDescriptor::new(Arc::new(t))))
- .unwrap();
-
- let column_desc = schema.column(0);
- let page_iterator = EmptyPageIterator::new(schema);
-
- let mut array_reader = PrimitiveArrayReader::<Int32Type>::new(
- Box::new(page_iterator),
- column_desc,
- None,
- )
- .unwrap();
-
- // expect no values to be read
- let array = array_reader.next_batch(50).unwrap();
- assert!(array.is_empty());
- }
-
- #[test]
- fn test_primitive_array_reader_data() {
- // Construct column schema
- let message_type = "
- message test_schema {
- REQUIRED INT32 leaf;
- }
- ";
-
- let schema = parse_message_type(message_type)
- .map(|t| Arc::new(SchemaDescriptor::new(Arc::new(t))))
- .unwrap();
-
- let column_desc = schema.column(0);
-
- // Construct page iterator
- {
- let mut data = Vec::new();
- let mut page_lists = Vec::new();
- make_column_chunks::<Int32Type>(
- column_desc.clone(),
- Encoding::PLAIN,
- 100,
- 1,
- 200,
- &mut Vec::new(),
- &mut Vec::new(),
- &mut data,
- &mut page_lists,
- true,
- 2,
- );
- let page_iterator =
- InMemoryPageIterator::new(schema, column_desc.clone(), page_lists);
-
- let mut array_reader = PrimitiveArrayReader::<Int32Type>::new(
- Box::new(page_iterator),
- column_desc,
- None,
- )
- .unwrap();
-
- // Read first 50 values, which are all from the first column chunk
- let array = array_reader.next_batch(50).unwrap();
- let array = array
- .as_any()
- .downcast_ref::<PrimitiveArray<ArrowInt32>>()
- .unwrap();
-
- assert_eq!(
- &PrimitiveArray::<ArrowInt32>::from(data[0..50].to_vec()),
- array
- );
-
- // Read next 100 values, the first 50 ones are from the first column chunk,
- // and the last 50 ones are from the second column chunk
- let array = array_reader.next_batch(100).unwrap();
- let array = array
- .as_any()
- .downcast_ref::<PrimitiveArray<ArrowInt32>>()
- .unwrap();
-
- assert_eq!(
- &PrimitiveArray::<ArrowInt32>::from(data[50..150].to_vec()),
- array
- );
-
- // Try to read 100 values, however there are only 50 values
- let array = array_reader.next_batch(100).unwrap();
- let array = array
- .as_any()
- .downcast_ref::<PrimitiveArray<ArrowInt32>>()
- .unwrap();
-
- assert_eq!(
- &PrimitiveArray::<ArrowInt32>::from(data[150..200].to_vec()),
- array
- );
- }
- }
-
- macro_rules! test_primitive_array_reader_one_type {
- ($arrow_parquet_type:ty, $physical_type:expr, $converted_type_str:expr, $result_arrow_type:ty, $result_arrow_cast_type:ty, $result_primitive_type:ty) => {{
- let message_type = format!(
- "
- message test_schema {{
- REQUIRED {:?} leaf ({});
- }}
- ",
- $physical_type, $converted_type_str
- );
- let schema = parse_message_type(&message_type)
- .map(|t| Arc::new(SchemaDescriptor::new(Arc::new(t))))
- .unwrap();
-
- let column_desc = schema.column(0);
-
- // Construct page iterator
- {
- let mut data = Vec::new();
- let mut page_lists = Vec::new();
- make_column_chunks::<$arrow_parquet_type>(
- column_desc.clone(),
- Encoding::PLAIN,
- 100,
- 1,
- 200,
- &mut Vec::new(),
- &mut Vec::new(),
- &mut data,
- &mut page_lists,
- true,
- 2,
- );
- let page_iterator = InMemoryPageIterator::new(
- schema.clone(),
- column_desc.clone(),
- page_lists,
- );
- let mut array_reader = PrimitiveArrayReader::<$arrow_parquet_type>::new(
- Box::new(page_iterator),
- column_desc.clone(),
- None,
- )
- .expect("Unable to get array reader");
-
- let array = array_reader
- .next_batch(50)
- .expect("Unable to get batch from reader");
-
- let result_data_type = <$result_arrow_type>::DATA_TYPE;
- let array = array
- .as_any()
- .downcast_ref::<PrimitiveArray<$result_arrow_type>>()
- .expect(
- format!(
- "Unable to downcast {:?} to {:?}",
- array.data_type(),
- result_data_type
- )
- .as_str(),
- );
-
- // create expected array as primitive, and cast to result type
- let expected = PrimitiveArray::<$result_arrow_cast_type>::from(
- data[0..50]
- .iter()
- .map(|x| *x as $result_primitive_type)
- .collect::<Vec<$result_primitive_type>>(),
- );
- let expected = Arc::new(expected) as ArrayRef;
- let expected = arrow::compute::cast(&expected, &result_data_type)
- .expect("Unable to cast expected array");
- assert_eq!(expected.data_type(), &result_data_type);
- let expected = expected
- .as_any()
- .downcast_ref::<PrimitiveArray<$result_arrow_type>>()
- .expect(
- format!(
- "Unable to downcast expected {:?} to {:?}",
- expected.data_type(),
- result_data_type
- )
- .as_str(),
- );
- assert_eq!(expected, array);
- }
- }};
- }
-
- #[test]
- fn test_primitive_array_reader_temporal_types() {
- test_primitive_array_reader_one_type!(
- Int32Type,
- PhysicalType::INT32,
- "DATE",
- ArrowDate32,
- ArrowInt32,
- i32
- );
- test_primitive_array_reader_one_type!(
- Int32Type,
- PhysicalType::INT32,
- "TIME_MILLIS",
- ArrowTime32MillisecondArray,
- ArrowInt32,
- i32
- );
- test_primitive_array_reader_one_type!(
- Int64Type,
- PhysicalType::INT64,
- "TIME_MICROS",
- ArrowTime64MicrosecondArray,
- ArrowInt64,
- i64
- );
- test_primitive_array_reader_one_type!(
- Int64Type,
- PhysicalType::INT64,
- "TIMESTAMP_MILLIS",
- ArrowTimestampMillisecondType,
- ArrowInt64,
- i64
- );
- test_primitive_array_reader_one_type!(
- Int64Type,
- PhysicalType::INT64,
- "TIMESTAMP_MICROS",
- ArrowTimestampMicrosecondType,
- ArrowInt64,
- i64
- );
- }
-
- #[test]
- fn test_primitive_array_reader_def_and_rep_levels() {
- // Construct column schema
- let message_type = "
- message test_schema {
- REPEATED Group test_mid {
- OPTIONAL INT32 leaf;
- }
- }
- ";
-
- let schema = parse_message_type(message_type)
- .map(|t| Arc::new(SchemaDescriptor::new(Arc::new(t))))
- .unwrap();
-
- let column_desc = schema.column(0);
-
- // Construct page iterator
- {
- let mut def_levels = Vec::new();
- let mut rep_levels = Vec::new();
- let mut page_lists = Vec::new();
- make_column_chunks::<Int32Type>(
- column_desc.clone(),
- Encoding::PLAIN,
- 100,
- 1,
- 200,
- &mut def_levels,
- &mut rep_levels,
- &mut Vec::new(),
- &mut page_lists,
- true,
- 2,
- );
-
- let page_iterator =
- InMemoryPageIterator::new(schema, column_desc.clone(), page_lists);
-
- let mut array_reader = PrimitiveArrayReader::<Int32Type>::new(
- Box::new(page_iterator),
- column_desc,
- None,
- )
- .unwrap();
-
- let mut accu_len: usize = 0;
-
- // Read first 50 values, which are all from the first column chunk
- let array = array_reader.next_batch(50).unwrap();
- assert_eq!(
- Some(&def_levels[accu_len..(accu_len + array.len())]),
- array_reader.get_def_levels()
- );
- assert_eq!(
- Some(&rep_levels[accu_len..(accu_len + array.len())]),
- array_reader.get_rep_levels()
- );
- accu_len += array.len();
-
- // Read next 100 values, the first 50 ones are from the first column chunk,
- // and the last 50 ones are from the second column chunk
- let array = array_reader.next_batch(100).unwrap();
- assert_eq!(
- Some(&def_levels[accu_len..(accu_len + array.len())]),
- array_reader.get_def_levels()
- );
- assert_eq!(
- Some(&rep_levels[accu_len..(accu_len + array.len())]),
- array_reader.get_rep_levels()
- );
- accu_len += array.len();
-
- // Try to read 100 values, however there are only 50 values
- let array = array_reader.next_batch(100).unwrap();
- assert_eq!(
- Some(&def_levels[accu_len..(accu_len + array.len())]),
- array_reader.get_def_levels()
- );
- assert_eq!(
- Some(&rep_levels[accu_len..(accu_len + array.len())]),
- array_reader.get_rep_levels()
- );
- }
- }
-
- #[test]
- fn test_complex_array_reader_no_pages() {
- let message_type = "
- message test_schema {
- REPEATED Group test_mid {
- OPTIONAL BYTE_ARRAY leaf (UTF8);
- }
- }
- ";
- let schema = parse_message_type(message_type)
- .map(|t| Arc::new(SchemaDescriptor::new(Arc::new(t))))
- .unwrap();
- let column_desc = schema.column(0);
- let pages: Vec<Vec<Page>> = Vec::new();
- let page_iterator = InMemoryPageIterator::new(schema, column_desc.clone(), pages);
-
- let converter = Utf8Converter::new(Utf8ArrayConverter {});
- let mut array_reader =
- ComplexObjectArrayReader::<ByteArrayType, Utf8Converter>::new(
- Box::new(page_iterator),
- column_desc,
- converter,
- None,
- )
- .unwrap();
-
- let values_per_page = 100; // this value is arbitrary in this test - the result should always be an array of 0 length
- let array = array_reader.next_batch(values_per_page).unwrap();
- assert_eq!(array.len(), 0);
- }
-
- #[test]
- fn test_complex_array_reader_def_and_rep_levels() {
- // Construct column schema
- let message_type = "
- message test_schema {
- REPEATED Group test_mid {
- OPTIONAL BYTE_ARRAY leaf (UTF8);
- }
- }
- ";
- let num_pages = 2;
- let values_per_page = 100;
- let str_base = "Hello World";
-
- let schema = parse_message_type(message_type)
- .map(|t| Arc::new(SchemaDescriptor::new(Arc::new(t))))
- .unwrap();
-
- let max_def_level = schema.column(0).max_def_level();
- let max_rep_level = schema.column(0).max_rep_level();
-
- assert_eq!(max_def_level, 2);
- assert_eq!(max_rep_level, 1);
-
- let mut rng = thread_rng();
- let column_desc = schema.column(0);
- let mut pages: Vec<Vec<Page>> = Vec::new();
-
- let mut rep_levels = Vec::with_capacity(num_pages * values_per_page);
- let mut def_levels = Vec::with_capacity(num_pages * values_per_page);
- let mut all_values = Vec::with_capacity(num_pages * values_per_page);
-
- for i in 0..num_pages {
- let mut values = Vec::with_capacity(values_per_page);
-
- for _ in 0..values_per_page {
- let def_level = rng.gen_range(0..max_def_level + 1);
- let rep_level = rng.gen_range(0..max_rep_level + 1);
- if def_level == max_def_level {
- let len = rng.gen_range(1..str_base.len());
- let slice = &str_base[..len];
- values.push(ByteArray::from(slice));
- all_values.push(Some(slice.to_string()));
- } else {
- all_values.push(None)
- }
- rep_levels.push(rep_level);
- def_levels.push(def_level)
- }
-
- let range = i * values_per_page..(i + 1) * values_per_page;
- let mut pb =
- DataPageBuilderImpl::new(column_desc.clone(), values.len() as u32, true);
-
- pb.add_rep_levels(max_rep_level, &rep_levels.as_slice()[range.clone()]);
- pb.add_def_levels(max_def_level, &def_levels.as_slice()[range]);
- pb.add_values::<ByteArrayType>(Encoding::PLAIN, values.as_slice());
-
- let data_page = pb.consume();
- pages.push(vec![data_page]);
- }
-
- let page_iterator = InMemoryPageIterator::new(schema, column_desc.clone(), pages);
-
- let converter = Utf8Converter::new(Utf8ArrayConverter {});
- let mut array_reader =
- ComplexObjectArrayReader::<ByteArrayType, Utf8Converter>::new(
- Box::new(page_iterator),
- column_desc,
- converter,
- None,
- )
- .unwrap();
-
- let mut accu_len: usize = 0;
-
- let array = array_reader.next_batch(values_per_page / 2).unwrap();
- assert_eq!(array.len(), values_per_page / 2);
- assert_eq!(
- Some(&def_levels[accu_len..(accu_len + array.len())]),
- array_reader.get_def_levels()
- );
- assert_eq!(
- Some(&rep_levels[accu_len..(accu_len + array.len())]),
- array_reader.get_rep_levels()
- );
- accu_len += array.len();
-
- // Read next values_per_page values, the first values_per_page/2 ones are from the first column chunk,
- // and the last values_per_page/2 ones are from the second column chunk
- let array = array_reader.next_batch(values_per_page).unwrap();
- assert_eq!(array.len(), values_per_page);
- assert_eq!(
- Some(&def_levels[accu_len..(accu_len + array.len())]),
- array_reader.get_def_levels()
- );
- assert_eq!(
- Some(&rep_levels[accu_len..(accu_len + array.len())]),
- array_reader.get_rep_levels()
- );
- let strings = array.as_any().downcast_ref::<StringArray>().unwrap();
- for i in 0..array.len() {
- if array.is_valid(i) {
- assert_eq!(
- all_values[i + accu_len].as_ref().unwrap().as_str(),
- strings.value(i)
- )
- } else {
- assert_eq!(all_values[i + accu_len], None)
- }
- }
- accu_len += array.len();
-
- // Try to read values_per_page values, however there are only values_per_page/2 values
- let array = array_reader.next_batch(values_per_page).unwrap();
- assert_eq!(array.len(), values_per_page / 2);
- assert_eq!(
- Some(&def_levels[accu_len..(accu_len + array.len())]),
- array_reader.get_def_levels()
- );
- assert_eq!(
- Some(&rep_levels[accu_len..(accu_len + array.len())]),
- array_reader.get_rep_levels()
- );
- }
-
- /// Array reader for test.
- struct InMemoryArrayReader {
- data_type: ArrowType,
- array: ArrayRef,
- def_levels: Option<Vec<i16>>,
- rep_levels: Option<Vec<i16>>,
- }
-
- impl InMemoryArrayReader {
- pub fn new(
- data_type: ArrowType,
- array: ArrayRef,
- def_levels: Option<Vec<i16>>,
- rep_levels: Option<Vec<i16>>,
- ) -> Self {
- Self {
- data_type,
- array,
- def_levels,
- rep_levels,
- }
- }
- }
-
- impl ArrayReader for InMemoryArrayReader {
- fn as_any(&self) -> &dyn Any {
- self
- }
-
- fn get_data_type(&self) -> &ArrowType {
- &self.data_type
- }
-
- fn next_batch(&mut self, _batch_size: usize) -> Result<ArrayRef> {
- Ok(self.array.clone())
- }
-
- fn get_def_levels(&self) -> Option<&[i16]> {
- self.def_levels.as_deref()
- }
-
- fn get_rep_levels(&self) -> Option<&[i16]> {
- self.rep_levels.as_deref()
- }
- }
-
- /// Iterator for testing reading empty columns
- struct EmptyPageIterator {
- schema: SchemaDescPtr,
- }
-
- impl EmptyPageIterator {
- fn new(schema: SchemaDescPtr) -> Self {
- EmptyPageIterator { schema }
- }
- }
-
- impl Iterator for EmptyPageIterator {
- type Item = Result<Box<dyn PageReader>>;
-
- fn next(&mut self) -> Option<Self::Item> {
- None
- }
- }
-
- impl PageIterator for EmptyPageIterator {
- fn schema(&mut self) -> Result<SchemaDescPtr> {
- Ok(self.schema.clone())
- }
-
- fn column_schema(&mut self) -> Result<ColumnDescPtr> {
- Ok(self.schema.column(0))
- }
- }
-
- #[test]
- fn test_struct_array_reader() {
- let array_1 = Arc::new(PrimitiveArray::<ArrowInt32>::from(vec![1, 2, 3, 4, 5]));
- let array_reader_1 = InMemoryArrayReader::new(
- ArrowType::Int32,
- array_1.clone(),
- Some(vec![0, 1, 2, 3, 1]),
- Some(vec![1, 1, 1, 1, 1]),
- );
-
- let array_2 = Arc::new(PrimitiveArray::<ArrowInt32>::from(vec![5, 4, 3, 2, 1]));
- let array_reader_2 = InMemoryArrayReader::new(
- ArrowType::Int32,
- array_2.clone(),
- Some(vec![0, 1, 3, 1, 2]),
- Some(vec![1, 1, 1, 1, 1]),
- );
-
- let struct_type = ArrowType::Struct(vec![
- Field::new("f1", array_1.data_type().clone(), true),
- Field::new("f2", array_2.data_type().clone(), true),
- ]);
-
- let mut struct_array_reader = StructArrayReader::new(
- struct_type,
- vec![Box::new(array_reader_1), Box::new(array_reader_2)],
- 1,
- 1,
- );
-
- let struct_array = struct_array_reader.next_batch(5).unwrap();
- let struct_array = struct_array.as_any().downcast_ref::<StructArray>().unwrap();
-
- assert_eq!(5, struct_array.len());
- assert_eq!(
- vec![true, false, false, false, false],
- (0..5)
- .map(|idx| struct_array.data_ref().is_null(idx))
- .collect::<Vec<bool>>()
- );
- assert_eq!(
- Some(vec![0, 1, 1, 1, 1].as_slice()),
- struct_array_reader.get_def_levels()
- );
- assert_eq!(
- Some(vec![1, 1, 1, 1, 1].as_slice()),
- struct_array_reader.get_rep_levels()
- );
- }
-
- #[test]
- fn test_create_array_reader() {
- let file = get_test_file("nulls.snappy.parquet");
- let file_reader = Arc::new(SerializedFileReader::new(file).unwrap());
-
- let file_metadata = file_reader.metadata().file_metadata();
- let arrow_schema = parquet_to_arrow_schema(
- file_metadata.schema_descr(),
- file_metadata.key_value_metadata(),
- )
- .unwrap();
-
- let array_reader = build_array_reader(
- file_reader.metadata().file_metadata().schema_descr_ptr(),
- arrow_schema,
- vec![0usize].into_iter(),
- file_reader,
- )
- .unwrap();
-
- // Create arrow types
- let arrow_type = ArrowType::Struct(vec![Field::new(
- "b_struct",
- ArrowType::Struct(vec![Field::new("b_c_int", ArrowType::Int32, true)]),
- true,
- )]);
-
- assert_eq!(array_reader.get_data_type(), &arrow_type);
- }
-
- #[test]
- fn test_list_array_reader() {
- // [[1, null, 2], null, [3, 4]]
- let array = Arc::new(PrimitiveArray::<ArrowInt32>::from(vec![
- Some(1),
- None,
- Some(2),
- None,
- Some(3),
- Some(4),
- ]));
- let item_array_reader = InMemoryArrayReader::new(
- ArrowType::Int32,
- array,
- Some(vec![3, 2, 3, 0, 3, 3]),
- Some(vec![0, 1, 1, 0, 0, 1]),
- );
-
- let mut list_array_reader = ListArrayReader::<i32>::new(
- Box::new(item_array_reader),
- ArrowType::List(Box::new(Field::new("item", ArrowType::Int32, true))),
- ArrowType::Int32,
- 1,
- 1,
- 0,
- 1,
- );
-
- let next_batch = list_array_reader.next_batch(1024).unwrap();
- let list_array = next_batch.as_any().downcast_ref::<ListArray>().unwrap();
-
- assert_eq!(3, list_array.len());
- // This passes as I expect
- assert_eq!(1, list_array.null_count());
-
- assert_eq!(
- list_array
- .value(0)
- .as_any()
- .downcast_ref::<PrimitiveArray<ArrowInt32>>()
- .unwrap(),
- &PrimitiveArray::<ArrowInt32>::from(vec![Some(1), None, Some(2)])
- );
-
- assert!(list_array.is_null(1));
-
- assert_eq!(
- list_array
- .value(2)
- .as_any()
- .downcast_ref::<PrimitiveArray<ArrowInt32>>()
- .unwrap(),
- &PrimitiveArray::<ArrowInt32>::from(vec![Some(3), Some(4)])
- );
- }
-
- #[test]
- fn test_large_list_array_reader() {
- // [[1, null, 2], null, [3, 4]]
- let array = Arc::new(PrimitiveArray::<ArrowInt32>::from(vec![
- Some(1),
- None,
- Some(2),
- None,
- Some(3),
- Some(4),
- ]));
- let item_array_reader = InMemoryArrayReader::new(
- ArrowType::Int32,
- array,
- Some(vec![3, 2, 3, 0, 3, 3]),
- Some(vec![0, 1, 1, 0, 0, 1]),
- );
-
- let mut list_array_reader = ListArrayReader::<i64>::new(
- Box::new(item_array_reader),
- ArrowType::LargeList(Box::new(Field::new("item", ArrowType::Int32, true))),
- ArrowType::Int32,
- 1,
- 1,
- 0,
- 1,
- );
-
- let next_batch = list_array_reader.next_batch(1024).unwrap();
- let list_array = next_batch
- .as_any()
- .downcast_ref::<LargeListArray>()
- .unwrap();
-
- assert_eq!(3, list_array.len());
-
- assert_eq!(
- list_array
- .value(0)
- .as_any()
- .downcast_ref::<PrimitiveArray<ArrowInt32>>()
- .unwrap(),
- &PrimitiveArray::<ArrowInt32>::from(vec![Some(1), None, Some(2)])
- );
-
- assert!(list_array.is_null(1));
-
- assert_eq!(
- list_array
- .value(2)
- .as_any()
- .downcast_ref::<PrimitiveArray<ArrowInt32>>()
- .unwrap(),
- &PrimitiveArray::<ArrowInt32>::from(vec![Some(3), Some(4)])
- );
- }
-}
diff --git a/parquet/src/arrow/arrow_array_reader.rs b/parquet/src/arrow/arrow_array_reader.rs
deleted file mode 100644
index c06d872..0000000
--- a/parquet/src/arrow/arrow_array_reader.rs
+++ /dev/null
@@ -1,1562 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-use super::array_reader::ArrayReader;
-use crate::arrow::schema::parquet_to_arrow_field;
-use crate::basic::Encoding;
-use crate::errors::{ParquetError, Result};
-use crate::{
- column::page::{Page, PageIterator},
- memory::ByteBufferPtr,
- schema::types::{ColumnDescPtr, ColumnDescriptor},
-};
-use arrow::{
- array::{ArrayRef, Int16Array},
- buffer::MutableBuffer,
- datatypes::{DataType as ArrowType, ToByteSlice},
-};
-use std::{any::Any, collections::VecDeque, marker::PhantomData};
-use std::{cell::RefCell, rc::Rc};
-
-struct UnzipIter<Source, Target, State> {
- shared_state: Rc<RefCell<State>>,
- select_item_buffer: fn(&mut State) -> &mut VecDeque<Target>,
- consume_source_item: fn(source_item: Source, state: &mut State) -> Target,
-}
-
-impl<Source, Target, State> UnzipIter<Source, Target, State> {
- fn new(
- shared_state: Rc<RefCell<State>>,
- item_buffer_selector: fn(&mut State) -> &mut VecDeque<Target>,
- source_item_consumer: fn(source_item: Source, state: &mut State) -> Target,
- ) -> Self {
- Self {
- shared_state,
- select_item_buffer: item_buffer_selector,
- consume_source_item: source_item_consumer,
- }
- }
-}
-
-trait UnzipIterState<T> {
- type SourceIter: Iterator<Item = T>;
- fn source_iter(&mut self) -> &mut Self::SourceIter;
-}
-
-impl<Source, Target, State: UnzipIterState<Source>> Iterator
- for UnzipIter<Source, Target, State>
-{
- type Item = Target;
-
- fn next(&mut self) -> Option<Self::Item> {
- let mut inner = self.shared_state.borrow_mut();
- // try to get one from the stored data
- (self.select_item_buffer)(&mut *inner)
- .pop_front()
- .or_else(||
- // nothing stored, we need a new element.
- inner.source_iter().next().map(|s| {
- (self.consume_source_item)(s, &mut inner)
- }))
- }
-}
-
-struct PageBufferUnzipIterState<V, L, It> {
- iter: It,
- value_iter_buffer: VecDeque<V>,
- def_level_iter_buffer: VecDeque<L>,
- rep_level_iter_buffer: VecDeque<L>,
-}
-
-impl<V, L, It: Iterator<Item = (V, L, L)>> UnzipIterState<(V, L, L)>
- for PageBufferUnzipIterState<V, L, It>
-{
- type SourceIter = It;
-
- #[inline]
- fn source_iter(&mut self) -> &mut Self::SourceIter {
- &mut self.iter
- }
-}
-
-type ValueUnzipIter<V, L, It> =
- UnzipIter<(V, L, L), V, PageBufferUnzipIterState<V, L, It>>;
-type LevelUnzipIter<V, L, It> =
- UnzipIter<(V, L, L), L, PageBufferUnzipIterState<V, L, It>>;
-type PageUnzipResult<V, L, It> = (
- ValueUnzipIter<V, L, It>,
- LevelUnzipIter<V, L, It>,
- LevelUnzipIter<V, L, It>,
-);
-
-fn unzip_iter<V, L, It: Iterator<Item = (V, L, L)>>(it: It) -> PageUnzipResult<V, L, It> {
- let shared_data = Rc::new(RefCell::new(PageBufferUnzipIterState {
- iter: it,
- value_iter_buffer: VecDeque::new(),
- def_level_iter_buffer: VecDeque::new(),
- rep_level_iter_buffer: VecDeque::new(),
- }));
-
- let value_iter = UnzipIter::new(
- shared_data.clone(),
- |state| &mut state.value_iter_buffer,
- |(v, d, r), state| {
- state.def_level_iter_buffer.push_back(d);
- state.rep_level_iter_buffer.push_back(r);
- v
- },
- );
-
- let def_level_iter = UnzipIter::new(
- shared_data.clone(),
- |state| &mut state.def_level_iter_buffer,
- |(v, d, r), state| {
- state.value_iter_buffer.push_back(v);
- state.rep_level_iter_buffer.push_back(r);
- d
- },
- );
-
- let rep_level_iter = UnzipIter::new(
- shared_data,
- |state| &mut state.rep_level_iter_buffer,
- |(v, d, r), state| {
- state.value_iter_buffer.push_back(v);
- state.def_level_iter_buffer.push_back(d);
- r
- },
- );
-
- (value_iter, def_level_iter, rep_level_iter)
-}
-
-pub trait ArrayConverter {
- fn convert_value_bytes(
- &self,
- value_decoder: &mut impl ValueDecoder,
- num_values: usize,
- ) -> Result<arrow::array::ArrayData>;
-}
-
-pub struct ArrowArrayReader<'a, C: ArrayConverter + 'a> {
- column_desc: ColumnDescPtr,
- data_type: ArrowType,
- def_level_decoder: Box<dyn ValueDecoder + 'a>,
- rep_level_decoder: Box<dyn ValueDecoder + 'a>,
- value_decoder: Box<dyn ValueDecoder + 'a>,
- last_def_levels: Option<Int16Array>,
- last_rep_levels: Option<Int16Array>,
- array_converter: C,
-}
-
-pub(crate) struct ColumnChunkContext {
- dictionary_values: Option<Vec<ByteBufferPtr>>,
-}
-
-impl ColumnChunkContext {
- fn new() -> Self {
- Self {
- dictionary_values: None,
- }
- }
-
- fn set_dictionary(&mut self, dictionary_values: Vec<ByteBufferPtr>) {
- self.dictionary_values = Some(dictionary_values);
- }
-}
-
-type PageDecoderTuple = (
- Box<dyn ValueDecoder>,
- Box<dyn ValueDecoder>,
- Box<dyn ValueDecoder>,
-);
-
-impl<'a, C: ArrayConverter + 'a> ArrowArrayReader<'a, C> {
- pub fn try_new<P: PageIterator + 'a>(
- column_chunk_iterator: P,
- column_desc: ColumnDescPtr,
- array_converter: C,
- arrow_type: Option<ArrowType>,
- ) -> Result<Self> {
- let data_type = match arrow_type {
- Some(t) => t,
- None => parquet_to_arrow_field(column_desc.as_ref())?
- .data_type()
- .clone(),
- };
- type PageIteratorItem = Result<(Page, Rc<RefCell<ColumnChunkContext>>)>;
- let page_iter = column_chunk_iterator
- // build iterator of pages across column chunks
- .flat_map(|x| -> Box<dyn Iterator<Item = PageIteratorItem>> {
- // attach column chunk context
- let context = Rc::new(RefCell::new(ColumnChunkContext::new()));
- match x {
- Ok(page_reader) => Box::new(
- page_reader.map(move |pr| pr.map(|p| (p, context.clone()))),
- ),
- // errors from reading column chunks / row groups are propagated to page level
- Err(e) => Box::new(std::iter::once(Err(e))),
- }
- });
- // capture a clone of column_desc in closure so that it can outlive current function
- let map_page_fn_factory = |column_desc: ColumnDescPtr| {
- move |x: Result<(Page, Rc<RefCell<ColumnChunkContext>>)>| {
- x.and_then(|(page, context)| {
- Self::map_page(page, context, column_desc.as_ref())
- })
- }
- };
- let map_page_fn = map_page_fn_factory(column_desc.clone());
- // map page iterator into tuple of buffer iterators for (values, def levels, rep levels)
- // errors from lower levels are surfaced through the value decoder iterator
- let decoder_iter = page_iter.map(map_page_fn).map(|x| match x {
- Ok(iter_tuple) => iter_tuple,
- // errors from reading pages are propagated to decoder iterator level
- Err(e) => Self::map_page_error(e),
- });
- // split tuple iterator into separate iterators for (values, def levels, rep levels)
- let (value_iter, def_level_iter, rep_level_iter) = unzip_iter(decoder_iter);
-
- Ok(Self {
- column_desc,
- data_type,
- def_level_decoder: Box::new(CompositeValueDecoder::new(def_level_iter)),
- rep_level_decoder: Box::new(CompositeValueDecoder::new(rep_level_iter)),
- value_decoder: Box::new(CompositeValueDecoder::new(value_iter)),
- last_def_levels: None,
- last_rep_levels: None,
- array_converter,
- })
- }
-
- #[inline]
- fn def_levels_available(column_desc: &ColumnDescriptor) -> bool {
- column_desc.max_def_level() > 0
- }
-
- #[inline]
- fn rep_levels_available(column_desc: &ColumnDescriptor) -> bool {
- column_desc.max_rep_level() > 0
- }
-
- fn map_page_error(err: ParquetError) -> PageDecoderTuple {
- (
- Box::new(<dyn ValueDecoder>::once(Err(err.clone()))),
- Box::new(<dyn ValueDecoder>::once(Err(err.clone()))),
- Box::new(<dyn ValueDecoder>::once(Err(err))),
- )
- }
-
- // Split Result<Page> into Result<(Iterator<Values>, Iterator<DefLevels>, Iterator<RepLevels>)>
- // this method could fail, e.g. if the page encoding is not supported
- fn map_page(
- page: Page,
- column_chunk_context: Rc<RefCell<ColumnChunkContext>>,
- column_desc: &ColumnDescriptor,
- ) -> Result<PageDecoderTuple> {
- use crate::encodings::levels::LevelDecoder;
- match page {
- Page::DictionaryPage {
- buf,
- num_values,
- encoding,
- ..
- } => {
- let mut column_chunk_context = column_chunk_context.borrow_mut();
- if column_chunk_context.dictionary_values.is_some() {
- return Err(general_err!(
- "Column chunk cannot have more than one dictionary"
- ));
- }
- // create plain decoder for dictionary values
- let mut dict_decoder = Self::get_dictionary_page_decoder(
- buf,
- num_values as usize,
- encoding,
- column_desc,
- )?;
- // decode and cache dictionary values
- let dictionary_values = dict_decoder.read_dictionary_values()?;
- column_chunk_context.set_dictionary(dictionary_values);
-
- // a dictionary page doesn't return any values
- Ok((
- Box::new(<dyn ValueDecoder>::empty()),
- Box::new(<dyn ValueDecoder>::empty()),
- Box::new(<dyn ValueDecoder>::empty()),
- ))
- }
- Page::DataPage {
- buf,
- num_values,
- encoding,
- def_level_encoding,
- rep_level_encoding,
- statistics: _,
- } => {
- let mut buffer_ptr = buf;
- // create rep level decoder iterator
- let rep_level_iter: Box<dyn ValueDecoder> =
- if Self::rep_levels_available(&column_desc) {
- let mut rep_decoder = LevelDecoder::v1(
- rep_level_encoding,
- column_desc.max_rep_level(),
- );
- let rep_level_byte_len =
- rep_decoder.set_data(num_values as usize, buffer_ptr.all());
- // advance buffer pointer
- buffer_ptr = buffer_ptr.start_from(rep_level_byte_len);
- Box::new(LevelValueDecoder::new(rep_decoder))
- } else {
- Box::new(<dyn ValueDecoder>::once(Err(ParquetError::General(
- "rep levels are not available".to_string(),
- ))))
- };
- // create def level decoder iterator
- let def_level_iter: Box<dyn ValueDecoder> =
- if Self::def_levels_available(&column_desc) {
- let mut def_decoder = LevelDecoder::v1(
- def_level_encoding,
- column_desc.max_def_level(),
- );
- let def_levels_byte_len =
- def_decoder.set_data(num_values as usize, buffer_ptr.all());
- // advance buffer pointer
- buffer_ptr = buffer_ptr.start_from(def_levels_byte_len);
- Box::new(LevelValueDecoder::new(def_decoder))
- } else {
- Box::new(<dyn ValueDecoder>::once(Err(ParquetError::General(
- "def levels are not available".to_string(),
- ))))
- };
- // create value decoder iterator
- let value_iter = Self::get_value_decoder(
- buffer_ptr,
- num_values as usize,
- encoding,
- column_desc,
- column_chunk_context,
- )?;
- Ok((value_iter, def_level_iter, rep_level_iter))
- }
- Page::DataPageV2 {
- buf,
- num_values,
- encoding,
- num_nulls: _,
- num_rows: _,
- def_levels_byte_len,
- rep_levels_byte_len,
- is_compressed: _,
- statistics: _,
- } => {
- let mut offset = 0;
- // create rep level decoder iterator
- let rep_level_iter: Box<dyn ValueDecoder> =
- if Self::rep_levels_available(&column_desc) {
- let rep_levels_byte_len = rep_levels_byte_len as usize;
- let mut rep_decoder =
- LevelDecoder::v2(column_desc.max_rep_level());
- rep_decoder.set_data_range(
- num_values as usize,
- &buf,
- offset,
- rep_levels_byte_len,
- );
- offset += rep_levels_byte_len;
- Box::new(LevelValueDecoder::new(rep_decoder))
- } else {
- Box::new(<dyn ValueDecoder>::once(Err(ParquetError::General(
- "rep levels are not available".to_string(),
- ))))
- };
- // create def level decoder iterator
- let def_level_iter: Box<dyn ValueDecoder> =
- if Self::def_levels_available(&column_desc) {
- let def_levels_byte_len = def_levels_byte_len as usize;
- let mut def_decoder =
- LevelDecoder::v2(column_desc.max_def_level());
- def_decoder.set_data_range(
- num_values as usize,
- &buf,
- offset,
- def_levels_byte_len,
- );
- offset += def_levels_byte_len;
- Box::new(LevelValueDecoder::new(def_decoder))
- } else {
- Box::new(<dyn ValueDecoder>::once(Err(ParquetError::General(
- "def levels are not available".to_string(),
- ))))
- };
-
- // create value decoder iterator
- let values_buffer = buf.start_from(offset);
- let value_iter = Self::get_value_decoder(
- values_buffer,
- num_values as usize,
- encoding,
- column_desc,
- column_chunk_context,
- )?;
- Ok((value_iter, def_level_iter, rep_level_iter))
- }
- }
- }
-
- fn get_dictionary_page_decoder(
- values_buffer: ByteBufferPtr,
- num_values: usize,
- mut encoding: Encoding,
- column_desc: &ColumnDescriptor,
- ) -> Result<Box<dyn DictionaryValueDecoder>> {
- if encoding == Encoding::PLAIN || encoding == Encoding::PLAIN_DICTIONARY {
- encoding = Encoding::RLE_DICTIONARY
- }
-
- if encoding == Encoding::RLE_DICTIONARY {
- Ok(
- Self::get_plain_value_decoder(values_buffer, num_values, column_desc)
- .into_dictionary_decoder(),
- )
- } else {
- Err(nyi_err!(
- "Invalid/Unsupported encoding type for dictionary: {}",
- encoding
- ))
- }
- }
-
- fn get_value_decoder(
- values_buffer: ByteBufferPtr,
- num_values: usize,
- mut encoding: Encoding,
- column_desc: &ColumnDescriptor,
- column_chunk_context: Rc<RefCell<ColumnChunkContext>>,
- ) -> Result<Box<dyn ValueDecoder>> {
- if encoding == Encoding::PLAIN_DICTIONARY {
- encoding = Encoding::RLE_DICTIONARY;
- }
-
- match encoding {
- Encoding::PLAIN => {
- Ok(
- Self::get_plain_value_decoder(values_buffer, num_values, column_desc)
- .into_value_decoder(),
- )
- }
- Encoding::RLE_DICTIONARY => {
- if column_chunk_context.borrow().dictionary_values.is_some() {
- let value_bit_len = Self::get_column_physical_bit_len(column_desc);
- let dictionary_decoder: Box<dyn ValueDecoder> = if value_bit_len == 0
- {
- Box::new(VariableLenDictionaryDecoder::new(
- column_chunk_context,
- values_buffer,
- num_values,
- ))
- } else {
- Box::new(FixedLenDictionaryDecoder::new(
- column_chunk_context,
- values_buffer,
- num_values,
- value_bit_len,
- ))
- };
- Ok(dictionary_decoder)
- } else {
- Err(general_err!("Dictionary values have not been initialized."))
- }
- }
- // Encoding::RLE => Box::new(RleValueDecoder::new()),
- // Encoding::DELTA_BINARY_PACKED => Box::new(DeltaBitPackDecoder::new()),
- // Encoding::DELTA_LENGTH_BYTE_ARRAY => Box::new(DeltaLengthByteArrayDecoder::new()),
- // Encoding::DELTA_BYTE_ARRAY => Box::new(DeltaByteArrayDecoder::new()),
- e => return Err(nyi_err!("Encoding {} is not supported", e)),
- }
- }
-
- fn get_column_physical_bit_len(column_desc: &ColumnDescriptor) -> usize {
- use crate::basic::Type as PhysicalType;
- // parquet only supports a limited number of physical types
- // later converters cast to a more specific arrow / logical type if necessary
- match column_desc.physical_type() {
- PhysicalType::BOOLEAN => 1,
- PhysicalType::INT32 | PhysicalType::FLOAT => 32,
- PhysicalType::INT64 | PhysicalType::DOUBLE => 64,
- PhysicalType::INT96 => 96,
- PhysicalType::BYTE_ARRAY => 0,
- PhysicalType::FIXED_LEN_BYTE_ARRAY => column_desc.type_length() as usize * 8,
- }
- }
-
- fn get_plain_value_decoder(
- values_buffer: ByteBufferPtr,
- num_values: usize,
- column_desc: &ColumnDescriptor,
- ) -> Box<dyn PlainValueDecoder> {
- let value_bit_len = Self::get_column_physical_bit_len(column_desc);
- if value_bit_len == 0 {
- Box::new(VariableLenPlainDecoder::new(values_buffer, num_values))
- } else {
- Box::new(FixedLenPlainDecoder::new(
- values_buffer,
- num_values,
- value_bit_len,
- ))
- }
- }
-
- fn build_level_array(
- level_decoder: &mut impl ValueDecoder,
- batch_size: usize,
- ) -> Result<Int16Array> {
- use arrow::datatypes::Int16Type;
- let level_converter = PrimitiveArrayConverter::<Int16Type>::new();
- let array_data =
- level_converter.convert_value_bytes(level_decoder, batch_size)?;
- Ok(Int16Array::from(array_data))
- }
-}
-
-impl<C: ArrayConverter> ArrayReader for ArrowArrayReader<'static, C> {
- fn as_any(&self) -> &dyn Any {
- self
- }
-
- fn get_data_type(&self) -> &ArrowType {
- &self.data_type
- }
-
- fn next_batch(&mut self, batch_size: usize) -> Result<ArrayRef> {
- if Self::rep_levels_available(&self.column_desc) {
- // read rep levels if available
- let rep_level_array =
- Self::build_level_array(&mut self.rep_level_decoder, batch_size)?;
- self.last_rep_levels = Some(rep_level_array);
- }
-
- // check if def levels are available
- let (values_to_read, null_bitmap_array) =
- if !Self::def_levels_available(&self.column_desc) {
- // if no def levels - just read (up to) batch_size values
- (batch_size, None)
- } else {
- // if def levels are available - they determine how many values will be read
- // decode def levels, return first error if any
- let def_level_array =
- Self::build_level_array(&mut self.def_level_decoder, batch_size)?;
- let def_level_count = def_level_array.len();
- // use eq_scalar to efficiently build null bitmap array from def levels
- let null_bitmap_array = arrow::compute::eq_scalar(
- &def_level_array,
- self.column_desc.max_def_level(),
- )?;
- self.last_def_levels = Some(def_level_array);
- // efficiently calculate values to read
- let values_to_read = null_bitmap_array
- .values()
- .count_set_bits_offset(0, def_level_count);
- let maybe_null_bitmap = if values_to_read != null_bitmap_array.len() {
- Some(null_bitmap_array)
- } else {
- // shortcut if no NULLs
- None
- };
- (values_to_read, maybe_null_bitmap)
- };
-
- // read a batch of values
- // converter only creates a no-null / all value array data
- let mut value_array_data = self
- .array_converter
- .convert_value_bytes(&mut self.value_decoder, values_to_read)?;
-
- if let Some(null_bitmap_array) = null_bitmap_array {
- // Only if def levels are available - insert null values efficiently using MutableArrayData.
- // This will require value bytes to be copied again, but converter requirements are reduced.
- // With a small number of NULLs, this will only be a few copies of large byte sequences.
- let actual_batch_size = null_bitmap_array.len();
- // use_nulls is false, because null_bitmap_array is already calculated and re-used
- let mut mutable = arrow::array::MutableArrayData::new(
- vec![&value_array_data],
- false,
- actual_batch_size,
- );
- // SlicesIterator slices only the true values, NULLs are inserted to fill any gaps
- arrow::compute::SlicesIterator::new(&null_bitmap_array).for_each(
- |(start, end)| {
- // the gap needs to be filled with NULLs
- if start > mutable.len() {
- let nulls_to_add = start - mutable.len();
- mutable.extend_nulls(nulls_to_add);
- }
- // fill values, adjust start and end with NULL count so far
- let nulls_added = mutable.null_count();
- mutable.extend(0, start - nulls_added, end - nulls_added);
- },
- );
- // any remaining part is NULLs
- if mutable.len() < actual_batch_size {
- let nulls_to_add = actual_batch_size - mutable.len();
- mutable.extend_nulls(nulls_to_add);
- }
-
- value_array_data = mutable
- .into_builder()
- .null_bit_buffer(null_bitmap_array.values().clone())
- .build();
- }
- let mut array = arrow::array::make_array(value_array_data);
- if array.data_type() != &self.data_type {
- // cast array to self.data_type if necessary
- array = arrow::compute::cast(&array, &self.data_type)?
- }
- Ok(array)
- }
-
- fn get_def_levels(&self) -> Option<&[i16]> {
- self.last_def_levels.as_ref().map(|x| x.values())
- }
-
- fn get_rep_levels(&self) -> Option<&[i16]> {
- self.last_rep_levels.as_ref().map(|x| x.values())
- }
-}
-
-use crate::encodings::rle::RleDecoder;
-
-pub trait ValueDecoder {
- fn read_value_bytes(
- &mut self,
- num_values: usize,
- read_bytes: &mut dyn FnMut(&[u8], usize),
- ) -> Result<usize>;
-}
-
-trait DictionaryValueDecoder {
- fn read_dictionary_values(&mut self) -> Result<Vec<ByteBufferPtr>>;
-}
-
-trait PlainValueDecoder: ValueDecoder + DictionaryValueDecoder {
- fn into_value_decoder(self: Box<Self>) -> Box<dyn ValueDecoder>;
- fn into_dictionary_decoder(self: Box<Self>) -> Box<dyn DictionaryValueDecoder>;
-}
-
-impl<T> PlainValueDecoder for T
-where
- T: ValueDecoder + DictionaryValueDecoder + 'static,
-{
- fn into_value_decoder(self: Box<T>) -> Box<dyn ValueDecoder> {
- self
- }
-
- fn into_dictionary_decoder(self: Box<T>) -> Box<dyn DictionaryValueDecoder> {
- self
- }
-}
-
-impl dyn ValueDecoder {
- fn empty() -> impl ValueDecoder {
- SingleValueDecoder::new(Ok(0))
- }
-
- fn once(value: Result<usize>) -> impl ValueDecoder {
- SingleValueDecoder::new(value)
- }
-}
-
-impl ValueDecoder for Box<dyn ValueDecoder> {
- #[inline]
- fn read_value_bytes(
- &mut self,
- num_values: usize,
- read_bytes: &mut dyn FnMut(&[u8], usize),
- ) -> Result<usize> {
- self.as_mut().read_value_bytes(num_values, read_bytes)
- }
-}
-
-struct SingleValueDecoder {
- value: Result<usize>,
-}
-
-impl SingleValueDecoder {
- fn new(value: Result<usize>) -> Self {
- Self { value }
- }
-}
-
-impl ValueDecoder for SingleValueDecoder {
- fn read_value_bytes(
- &mut self,
- _num_values: usize,
- _read_bytes: &mut dyn FnMut(&[u8], usize),
- ) -> Result<usize> {
- self.value.clone()
- }
-}
-
-struct CompositeValueDecoder<I: Iterator<Item = Box<dyn ValueDecoder>>> {
- current_decoder: Option<Box<dyn ValueDecoder>>,
- decoder_iter: I,
-}
-
-impl<I: Iterator<Item = Box<dyn ValueDecoder>>> CompositeValueDecoder<I> {
- fn new(mut decoder_iter: I) -> Self {
- let current_decoder = decoder_iter.next();
- Self {
- current_decoder,
- decoder_iter,
- }
- }
-}
-
-impl<I: Iterator<Item = Box<dyn ValueDecoder>>> ValueDecoder
- for CompositeValueDecoder<I>
-{
- fn read_value_bytes(
- &mut self,
- num_values: usize,
- read_bytes: &mut dyn FnMut(&[u8], usize),
- ) -> Result<usize> {
- let mut values_to_read = num_values;
- while values_to_read > 0 {
- let value_decoder = match self.current_decoder.as_mut() {
- Some(d) => d,
- // no more decoders
- None => break,
- };
- while values_to_read > 0 {
- let values_read =
- value_decoder.read_value_bytes(values_to_read, read_bytes)?;
- if values_read > 0 {
- values_to_read -= values_read;
- } else {
- // no more values in current decoder
- self.current_decoder = self.decoder_iter.next();
- break;
- }
- }
- }
-
- Ok(num_values - values_to_read)
- }
-}
-
-struct LevelValueDecoder {
- level_decoder: crate::encodings::levels::LevelDecoder,
- level_value_buffer: Vec<i16>,
-}
-
-impl LevelValueDecoder {
- fn new(level_decoder: crate::encodings::levels::LevelDecoder) -> Self {
- Self {
- level_decoder,
- level_value_buffer: vec![0i16; 2048],
- }
- }
-}
-
-impl ValueDecoder for LevelValueDecoder {
- fn read_value_bytes(
- &mut self,
- num_values: usize,
- read_bytes: &mut dyn FnMut(&[u8], usize),
- ) -> Result<usize> {
- let value_size = std::mem::size_of::<i16>();
- let mut total_values_read = 0;
- while total_values_read < num_values {
- let values_to_read = std::cmp::min(
- num_values - total_values_read,
- self.level_value_buffer.len(),
- );
- let values_read = match self
- .level_decoder
- .get(&mut self.level_value_buffer[..values_to_read])
- {
- Ok(values_read) => values_read,
- Err(e) => return Err(e),
- };
- if values_read > 0 {
- let level_value_bytes =
- &self.level_value_buffer.to_byte_slice()[..values_read * value_size];
- read_bytes(level_value_bytes, values_read);
- total_values_read += values_read;
- } else {
- break;
- }
- }
- Ok(total_values_read)
- }
-}
-
-pub(crate) struct FixedLenPlainDecoder {
- data: ByteBufferPtr,
- num_values: usize,
- value_bit_len: usize,
-}
-
-impl FixedLenPlainDecoder {
- pub(crate) fn new(
- data: ByteBufferPtr,
- num_values: usize,
- value_bit_len: usize,
- ) -> Self {
- Self {
- data,
- num_values,
- value_bit_len,
- }
- }
-}
-
-impl DictionaryValueDecoder for FixedLenPlainDecoder {
- fn read_dictionary_values(&mut self) -> Result<Vec<ByteBufferPtr>> {
- let value_byte_len = self.value_bit_len / 8;
- let available_values = self.data.len() / value_byte_len;
- let values_to_read = std::cmp::min(available_values, self.num_values);
- let byte_len = values_to_read * value_byte_len;
- let values = vec![self.data.range(0, byte_len)];
- self.num_values = 0;
- self.data.set_range(self.data.start(), 0);
- Ok(values)
- }
-}
-
-impl ValueDecoder for FixedLenPlainDecoder {
- fn read_value_bytes(
- &mut self,
- num_values: usize,
- read_bytes: &mut dyn FnMut(&[u8], usize),
- ) -> Result<usize> {
- let available_values = self.data.len() * 8 / self.value_bit_len;
- if available_values > 0 {
- let values_to_read = std::cmp::min(available_values, num_values);
- let byte_len = values_to_read * self.value_bit_len / 8;
- read_bytes(&self.data.data()[..byte_len], values_to_read);
- self.data
- .set_range(self.data.start() + byte_len, self.data.len() - byte_len);
- Ok(values_to_read)
- } else {
- Ok(0)
- }
- }
-}
-
-pub(crate) struct VariableLenPlainDecoder {
- data: ByteBufferPtr,
- num_values: usize,
- position: usize,
-}
-
-impl VariableLenPlainDecoder {
- pub(crate) fn new(data: ByteBufferPtr, num_values: usize) -> Self {
- Self {
- data,
- num_values,
- position: 0,
- }
- }
-}
-
-impl DictionaryValueDecoder for VariableLenPlainDecoder {
- fn read_dictionary_values(&mut self) -> Result<Vec<ByteBufferPtr>> {
- const LEN_SIZE: usize = std::mem::size_of::<u32>();
- let data = self.data.data();
- let data_len = data.len();
- let values_to_read = self.num_values;
- let mut values = Vec::with_capacity(values_to_read);
- let mut values_read = 0;
- while self.position < data_len && values_read < values_to_read {
- let len: usize =
- read_num_bytes!(u32, LEN_SIZE, data[self.position..]) as usize;
- self.position += LEN_SIZE;
- if data_len < self.position + len {
- return Err(eof_err!("Not enough bytes to decode"));
- }
- values.push(self.data.range(self.position, len));
- self.position += len;
- values_read += 1;
- }
- self.num_values -= values_read;
- Ok(values)
- }
-}
-
-impl ValueDecoder for VariableLenPlainDecoder {
- fn read_value_bytes(
- &mut self,
- num_values: usize,
- read_bytes: &mut dyn FnMut(&[u8], usize),
- ) -> Result<usize> {
- const LEN_SIZE: usize = std::mem::size_of::<u32>();
- let data = self.data.data();
- let data_len = data.len();
- let values_to_read = std::cmp::min(self.num_values, num_values);
- let mut values_read = 0;
- while self.position < data_len && values_read < values_to_read {
- let len: usize =
- read_num_bytes!(u32, LEN_SIZE, data[self.position..]) as usize;
- self.position += LEN_SIZE;
- if data_len < self.position + len {
- return Err(eof_err!("Not enough bytes to decode"));
- }
- read_bytes(&data[self.position..][..len], 1);
- self.position += len;
- values_read += 1;
- }
- self.num_values -= values_read;
- Ok(values_read)
- }
-}
-
-pub(crate) struct FixedLenDictionaryDecoder {
- context_ref: Rc<RefCell<ColumnChunkContext>>,
- key_data_bufer: ByteBufferPtr,
- num_values: usize,
- rle_decoder: RleDecoder,
- value_byte_len: usize,
- keys_buffer: Vec<i32>,
-}
-
-impl FixedLenDictionaryDecoder {
- pub(crate) fn new(
- column_chunk_context: Rc<RefCell<ColumnChunkContext>>,
- key_data_bufer: ByteBufferPtr,
- num_values: usize,
- value_bit_len: usize,
- ) -> Self {
- assert!(
- value_bit_len % 8 == 0,
- "value_bit_size must be a multiple of 8"
- );
- // First byte in `data` is bit width
- let bit_width = key_data_bufer.data()[0];
- let mut rle_decoder = RleDecoder::new(bit_width);
- rle_decoder.set_data(key_data_bufer.start_from(1));
-
- Self {
- context_ref: column_chunk_context,
- key_data_bufer,
- num_values,
- rle_decoder,
- value_byte_len: value_bit_len / 8,
- keys_buffer: vec![0; 2048],
- }
- }
-}
-
-impl ValueDecoder for FixedLenDictionaryDecoder {
- fn read_value_bytes(
- &mut self,
- num_values: usize,
- read_bytes: &mut dyn FnMut(&[u8], usize),
- ) -> Result<usize> {
- if self.num_values == 0 {
- return Ok(0);
- }
- let context = self.context_ref.borrow();
- let values = context.dictionary_values.as_ref().unwrap();
- let input_value_bytes = values[0].data();
- // read no more than available values or requested values
- let values_to_read = std::cmp::min(self.num_values, num_values);
- let mut values_read = 0;
- while values_read < values_to_read {
- // read values in batches of up to self.keys_buffer.len()
- let keys_to_read =
- std::cmp::min(values_to_read - values_read, self.keys_buffer.len());
- let keys_read = match self
- .rle_decoder
- .get_batch(&mut self.keys_buffer[..keys_to_read])
- {
- Ok(keys_read) => keys_read,
- Err(e) => return Err(e),
- };
- if keys_read == 0 {
- self.num_values = 0;
- return Ok(values_read);
- }
- for i in 0..keys_read {
- let key = self.keys_buffer[i] as usize;
- read_bytes(
- &input_value_bytes[key * self.value_byte_len..]
- [..self.value_byte_len],
- 1,
- );
- }
- values_read += keys_read;
- }
- self.num_values -= values_read;
- Ok(values_read)
- }
-}
-
-pub(crate) struct VariableLenDictionaryDecoder {
- context_ref: Rc<RefCell<ColumnChunkContext>>,
- key_data_bufer: ByteBufferPtr,
- num_values: usize,
- rle_decoder: RleDecoder,
- keys_buffer: Vec<i32>,
-}
-
-impl VariableLenDictionaryDecoder {
- pub(crate) fn new(
- column_chunk_context: Rc<RefCell<ColumnChunkContext>>,
- key_data_bufer: ByteBufferPtr,
- num_values: usize,
- ) -> Self {
- // First byte in `data` is bit width
- let bit_width = key_data_bufer.data()[0];
- let mut rle_decoder = RleDecoder::new(bit_width);
- rle_decoder.set_data(key_data_bufer.start_from(1));
-
- Self {
- context_ref: column_chunk_context,
- key_data_bufer,
- num_values,
- rle_decoder,
- keys_buffer: vec![0; 2048],
- }
- }
-}
-
-impl ValueDecoder for VariableLenDictionaryDecoder {
- fn read_value_bytes(
- &mut self,
- num_values: usize,
- read_bytes: &mut dyn FnMut(&[u8], usize),
- ) -> Result<usize> {
- if self.num_values == 0 {
- return Ok(0);
- }
- let context = self.context_ref.borrow();
- let values = context.dictionary_values.as_ref().unwrap();
- let values_to_read = std::cmp::min(self.num_values, num_values);
- let mut values_read = 0;
- while values_read < values_to_read {
- // read values in batches of up to self.keys_buffer.len()
- let keys_to_read =
- std::cmp::min(values_to_read - values_read, self.keys_buffer.len());
- let keys_read = match self
- .rle_decoder
- .get_batch(&mut self.keys_buffer[..keys_to_read])
- {
- Ok(keys_read) => keys_read,
- Err(e) => return Err(e),
- };
- if keys_read == 0 {
- self.num_values = 0;
- return Ok(values_read);
- }
- for i in 0..keys_read {
- let key = self.keys_buffer[i] as usize;
- read_bytes(values[key].data(), 1);
- }
- values_read += keys_read;
- }
- self.num_values -= values_read;
- Ok(values_read)
- }
-}
-
-use arrow::datatypes::ArrowPrimitiveType;
-
-pub struct PrimitiveArrayConverter<T: ArrowPrimitiveType> {
- _phantom_data: PhantomData<T>,
-}
-
-impl<T: ArrowPrimitiveType> PrimitiveArrayConverter<T> {
- pub fn new() -> Self {
- Self {
- _phantom_data: PhantomData,
- }
- }
-}
-
-impl<T: ArrowPrimitiveType> ArrayConverter for PrimitiveArrayConverter<T> {
- fn convert_value_bytes(
- &self,
- value_decoder: &mut impl ValueDecoder,
- num_values: usize,
- ) -> Result<arrow::array::ArrayData> {
- let value_size = T::get_byte_width();
- let values_byte_capacity = num_values * value_size;
- let mut values_buffer = MutableBuffer::new(values_byte_capacity);
-
- value_decoder.read_value_bytes(num_values, &mut |value_bytes, _| {
- values_buffer.extend_from_slice(value_bytes);
- })?;
-
- // calculate actual data_len, which may be different from the iterator's upper bound
- let value_count = values_buffer.len() / value_size;
- let array_data = arrow::array::ArrayData::builder(T::DATA_TYPE)
- .len(value_count)
- .add_buffer(values_buffer.into())
- .build();
- Ok(array_data)
- }
-}
-
-pub struct StringArrayConverter {}
-
-impl StringArrayConverter {
- pub fn new() -> Self {
- Self {}
- }
-}
-
-impl ArrayConverter for StringArrayConverter {
- fn convert_value_bytes(
- &self,
- value_decoder: &mut impl ValueDecoder,
- num_values: usize,
- ) -> Result<arrow::array::ArrayData> {
- use arrow::datatypes::ArrowNativeType;
- let offset_size = std::mem::size_of::<i32>();
- let mut offsets_buffer = MutableBuffer::new((num_values + 1) * offset_size);
- // allocate initial capacity of 1 byte for each item
- let values_byte_capacity = num_values;
- let mut values_buffer = MutableBuffer::new(values_byte_capacity);
-
- let mut length_so_far = i32::default();
- offsets_buffer.push(length_so_far);
-
- value_decoder.read_value_bytes(num_values, &mut |value_bytes, values_read| {
- debug_assert_eq!(
- values_read, 1,
- "offset length value buffers can only contain bytes for a single value"
- );
- length_so_far +=
- <i32 as ArrowNativeType>::from_usize(value_bytes.len()).unwrap();
- // this should be safe because a ValueDecoder should not read more than num_values
- unsafe {
- offsets_buffer.push_unchecked(length_so_far);
- }
- values_buffer.extend_from_slice(value_bytes);
- })?;
- // calculate actual data_len, which may be different from the iterator's upper bound
- let data_len = (offsets_buffer.len() / offset_size) - 1;
- let array_data = arrow::array::ArrayData::builder(ArrowType::Utf8)
- .len(data_len)
- .add_buffer(offsets_buffer.into())
- .add_buffer(values_buffer.into())
- .build();
- Ok(array_data)
- }
-}
-
-#[cfg(test)]
-mod tests {
- use super::*;
- use crate::column::page::Page;
- use crate::data_type::ByteArray;
- use crate::data_type::ByteArrayType;
- use crate::schema::parser::parse_message_type;
- use crate::schema::types::SchemaDescriptor;
- use crate::util::test_common::page_util::{
- DataPageBuilder, DataPageBuilderImpl, InMemoryPageIterator,
- };
- use crate::{
- basic::Encoding, column::page::PageReader, schema::types::SchemaDescPtr,
- };
- use arrow::array::{PrimitiveArray, StringArray};
- use arrow::datatypes::Int32Type as ArrowInt32;
- use rand::{distributions::uniform::SampleUniform, thread_rng, Rng};
- use std::sync::Arc;
-
- /// Iterator for testing reading empty columns
- struct EmptyPageIterator {
- schema: SchemaDescPtr,
- }
-
- impl EmptyPageIterator {
- fn new(schema: SchemaDescPtr) -> Self {
- EmptyPageIterator { schema }
- }
- }
-
- impl Iterator for EmptyPageIterator {
- type Item = Result<Box<dyn PageReader>>;
-
- fn next(&mut self) -> Option<Self::Item> {
- None
- }
- }
-
- impl PageIterator for EmptyPageIterator {
- fn schema(&mut self) -> Result<SchemaDescPtr> {
- Ok(self.schema.clone())
- }
-
- fn column_schema(&mut self) -> Result<ColumnDescPtr> {
- Ok(self.schema.column(0))
- }
- }
-
- #[test]
- fn test_array_reader_empty_pages() {
- // Construct column schema
- let message_type = "
- message test_schema {
- REQUIRED INT32 leaf;
- }
- ";
-
- let schema = parse_message_type(message_type)
- .map(|t| Arc::new(SchemaDescriptor::new(Arc::new(t))))
- .unwrap();
-
- let column_desc = schema.column(0);
- let page_iterator = EmptyPageIterator::new(schema);
-
- let converter = PrimitiveArrayConverter::<arrow::datatypes::Int32Type>::new();
- let mut array_reader =
- ArrowArrayReader::try_new(page_iterator, column_desc, converter, None)
- .unwrap();
-
- // expect no values to be read
- let array = array_reader.next_batch(50).unwrap();
- assert!(array.is_empty());
- }
-
- fn make_column_chunks<T: crate::data_type::DataType>(
- column_desc: ColumnDescPtr,
- encoding: Encoding,
- num_levels: usize,
- min_value: T::T,
- max_value: T::T,
- def_levels: &mut Vec<i16>,
- rep_levels: &mut Vec<i16>,
- values: &mut Vec<T::T>,
- page_lists: &mut Vec<Vec<Page>>,
- use_v2: bool,
- num_chunks: usize,
- ) where
- T::T: PartialOrd + SampleUniform + Copy,
- {
- for _i in 0..num_chunks {
- let mut pages = VecDeque::new();
- let mut data = Vec::new();
- let mut page_def_levels = Vec::new();
- let mut page_rep_levels = Vec::new();
-
- crate::util::test_common::make_pages::<T>(
- column_desc.clone(),
- encoding,
- 1,
- num_levels,
- min_value,
- max_value,
- &mut page_def_levels,
- &mut page_rep_levels,
- &mut data,
- &mut pages,
- use_v2,
- );
-
- def_levels.append(&mut page_def_levels);
- rep_levels.append(&mut page_rep_levels);
- values.append(&mut data);
- page_lists.push(Vec::from(pages));
- }
- }
-
- #[test]
- fn test_primitive_array_reader_data() {
- // Construct column schema
- let message_type = "
- message test_schema {
- REQUIRED INT32 leaf;
- }
- ";
-
- let schema = parse_message_type(message_type)
- .map(|t| Arc::new(SchemaDescriptor::new(Arc::new(t))))
- .unwrap();
-
- let column_desc = schema.column(0);
-
- // Construct page iterator
- {
- let mut data = Vec::new();
- let mut page_lists = Vec::new();
- make_column_chunks::<crate::data_type::Int32Type>(
- column_desc.clone(),
- Encoding::PLAIN,
- 100,
- 1,
- 200,
- &mut Vec::new(),
- &mut Vec::new(),
- &mut data,
- &mut page_lists,
- true,
- 2,
- );
- let page_iterator =
- InMemoryPageIterator::new(schema, column_desc.clone(), page_lists);
-
- let converter = PrimitiveArrayConverter::<arrow::datatypes::Int32Type>::new();
- let mut array_reader =
- ArrowArrayReader::try_new(page_iterator, column_desc, converter, None)
- .unwrap();
-
- // Read first 50 values, which are all from the first column chunk
- let array = array_reader.next_batch(50).unwrap();
- let array = array
- .as_any()
- .downcast_ref::<PrimitiveArray<ArrowInt32>>()
- .unwrap();
-
- assert_eq!(
- &PrimitiveArray::<ArrowInt32>::from(data[0..50].to_vec()),
- array
- );
-
- // Read next 100 values, the first 50 ones are from the first column chunk,
- // and the last 50 ones are from the second column chunk
- let array = array_reader.next_batch(100).unwrap();
- let array = array
- .as_any()
- .downcast_ref::<PrimitiveArray<ArrowInt32>>()
- .unwrap();
-
- assert_eq!(
- &PrimitiveArray::<ArrowInt32>::from(data[50..150].to_vec()),
- array
- );
-
- // Try to read 100 values, however there are only 50 values
- let array = array_reader.next_batch(100).unwrap();
- let array = array
- .as_any()
- .downcast_ref::<PrimitiveArray<ArrowInt32>>()
- .unwrap();
-
- assert_eq!(
- &PrimitiveArray::<ArrowInt32>::from(data[150..200].to_vec()),
- array
- );
- }
- }
-
- #[test]
- fn test_primitive_array_reader_def_and_rep_levels() {
- // Construct column schema
- let message_type = "
- message test_schema {
- REPEATED Group test_mid {
- OPTIONAL INT32 leaf;
- }
- }
- ";
-
- let schema = parse_message_type(message_type)
- .map(|t| Arc::new(SchemaDescriptor::new(Arc::new(t))))
- .unwrap();
-
- let column_desc = schema.column(0);
-
- // Construct page iterator
- {
- let mut def_levels = Vec::new();
- let mut rep_levels = Vec::new();
- let mut page_lists = Vec::new();
- make_column_chunks::<crate::data_type::Int32Type>(
- column_desc.clone(),
- Encoding::PLAIN,
- 100,
- 1,
- 200,
- &mut def_levels,
- &mut rep_levels,
- &mut Vec::new(),
- &mut page_lists,
- true,
- 2,
- );
-
- let page_iterator =
- InMemoryPageIterator::new(schema, column_desc.clone(), page_lists);
-
- let converter = PrimitiveArrayConverter::<arrow::datatypes::Int32Type>::new();
- let mut array_reader =
- ArrowArrayReader::try_new(page_iterator, column_desc, converter, None)
- .unwrap();
-
- let mut accu_len: usize = 0;
-
- // Read first 50 values, which are all from the first column chunk
- let array = array_reader.next_batch(50).unwrap();
- assert_eq!(
- Some(&def_levels[accu_len..(accu_len + array.len())]),
- array_reader.get_def_levels()
- );
- assert_eq!(
- Some(&rep_levels[accu_len..(accu_len + array.len())]),
- array_reader.get_rep_levels()
- );
- accu_len += array.len();
-
- // Read next 100 values, the first 50 ones are from the first column chunk,
- // and the last 50 ones are from the second column chunk
- let array = array_reader.next_batch(100).unwrap();
- assert_eq!(
- Some(&def_levels[accu_len..(accu_len + array.len())]),
- array_reader.get_def_levels()
- );
- assert_eq!(
- Some(&rep_levels[accu_len..(accu_len + array.len())]),
- array_reader.get_rep_levels()
- );
- accu_len += array.len();
-
- // Try to read 100 values, however there are only 50 values
- let array = array_reader.next_batch(100).unwrap();
- assert_eq!(
- Some(&def_levels[accu_len..(accu_len + array.len())]),
- array_reader.get_def_levels()
- );
- assert_eq!(
- Some(&rep_levels[accu_len..(accu_len + array.len())]),
- array_reader.get_rep_levels()
- );
-
- assert_eq!(accu_len + array.len(), 200);
- }
- }
-
- #[test]
- fn test_arrow_array_reader_string() {
- // Construct column schema
- let message_type = "
- message test_schema {
- REPEATED Group test_mid {
- OPTIONAL BYTE_ARRAY leaf (UTF8);
- }
- }
- ";
- let num_pages = 2;
- let values_per_page = 100;
- let str_base = "Hello World";
-
- let schema = parse_message_type(message_type)
- .map(|t| Arc::new(SchemaDescriptor::new(Arc::new(t))))
- .unwrap();
- let column_desc = schema.column(0);
- let max_def_level = column_desc.max_def_level();
- let max_rep_level = column_desc.max_rep_level();
-
- assert_eq!(max_def_level, 2);
- assert_eq!(max_rep_level, 1);
-
- let mut rng = thread_rng();
- let mut pages: Vec<Vec<Page>> = Vec::new();
-
- let mut rep_levels = Vec::with_capacity(num_pages * values_per_page);
- let mut def_levels = Vec::with_capacity(num_pages * values_per_page);
- let mut all_values = Vec::with_capacity(num_pages * values_per_page);
-
- for i in 0..num_pages {
- let mut values = Vec::with_capacity(values_per_page);
-
- for _ in 0..values_per_page {
- let def_level = rng.gen_range(0..max_def_level + 1);
- let rep_level = rng.gen_range(0..max_rep_level + 1);
- if def_level == max_def_level {
- let len = rng.gen_range(1..str_base.len());
- let slice = &str_base[..len];
- values.push(ByteArray::from(slice));
- all_values.push(Some(slice.to_string()));
- } else {
- all_values.push(None)
- }
- rep_levels.push(rep_level);
- def_levels.push(def_level)
- }
-
- let range = i * values_per_page..(i + 1) * values_per_page;
- let mut pb =
- DataPageBuilderImpl::new(column_desc.clone(), values.len() as u32, true);
-
- pb.add_rep_levels(max_rep_level, &rep_levels.as_slice()[range.clone()]);
- pb.add_def_levels(max_def_level, &def_levels.as_slice()[range]);
- pb.add_values::<ByteArrayType>(Encoding::PLAIN, values.as_slice());
-
- let data_page = pb.consume();
- pages.push(vec![data_page]);
- }
-
- let page_iterator = InMemoryPageIterator::new(schema, column_desc.clone(), pages);
- let converter = StringArrayConverter::new();
- let mut array_reader =
- ArrowArrayReader::try_new(page_iterator, column_desc, converter, None)
- .unwrap();
-
- let mut accu_len: usize = 0;
-
- let array = array_reader.next_batch(values_per_page / 2).unwrap();
- assert_eq!(array.len(), values_per_page / 2);
- assert_eq!(
- Some(&def_levels[accu_len..(accu_len + array.len())]),
- array_reader.get_def_levels()
- );
- assert_eq!(
- Some(&rep_levels[accu_len..(accu_len + array.len())]),
- array_reader.get_rep_levels()
- );
- accu_len += array.len();
-
- // Read next values_per_page values, the first values_per_page/2 ones are from the first column chunk,
- // and the last values_per_page/2 ones are from the second column chunk
- let array = array_reader.next_batch(values_per_page).unwrap();
- assert_eq!(array.len(), values_per_page);
- assert_eq!(
- Some(&def_levels[accu_len..(accu_len + array.len())]),
- array_reader.get_def_levels()
- );
- assert_eq!(
- Some(&rep_levels[accu_len..(accu_len + array.len())]),
- array_reader.get_rep_levels()
- );
- let strings = array.as_any().downcast_ref::<StringArray>().unwrap();
- for i in 0..array.len() {
- if array.is_valid(i) {
- assert_eq!(
- all_values[i + accu_len].as_ref().unwrap().as_str(),
- strings.value(i)
- )
- } else {
- assert_eq!(all_values[i + accu_len], None)
- }
- }
- accu_len += array.len();
-
- // Try to read values_per_page values, however there are only values_per_page/2 values
- let array = array_reader.next_batch(values_per_page).unwrap();
- assert_eq!(array.len(), values_per_page / 2);
- assert_eq!(
- Some(&def_levels[accu_len..(accu_len + array.len())]),
- array_reader.get_def_levels()
- );
- assert_eq!(
- Some(&rep_levels[accu_len..(accu_len + array.len())]),
- array_reader.get_rep_levels()
- );
- }
-}
diff --git a/parquet/src/arrow/arrow_reader.rs b/parquet/src/arrow/arrow_reader.rs
deleted file mode 100644
index 83fb0a2..0000000
--- a/parquet/src/arrow/arrow_reader.rs
+++ /dev/null
@@ -1,671 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-//! Contains reader which reads parquet data into arrow array.
-
-use crate::arrow::array_reader::{build_array_reader, ArrayReader, StructArrayReader};
-use crate::arrow::schema::parquet_to_arrow_schema;
-use crate::arrow::schema::{
- parquet_to_arrow_schema_by_columns, parquet_to_arrow_schema_by_root_columns,
-};
-use crate::errors::{ParquetError, Result};
-use crate::file::metadata::ParquetMetaData;
-use crate::file::reader::FileReader;
-use arrow::datatypes::{DataType as ArrowType, Schema, SchemaRef};
-use arrow::error::Result as ArrowResult;
-use arrow::record_batch::{RecordBatch, RecordBatchReader};
-use arrow::{array::StructArray, error::ArrowError};
-use std::sync::Arc;
-
-/// Arrow reader api.
-/// With this api, user can get arrow schema from parquet file, and read parquet data
-/// into arrow arrays.
-pub trait ArrowReader {
- type RecordReader: RecordBatchReader;
-
- /// Read parquet schema and convert it into arrow schema.
- fn get_schema(&mut self) -> Result<Schema>;
-
- /// Read parquet schema and convert it into arrow schema.
- /// This schema only includes columns identified by `column_indices`.
- /// To select leaf columns (i.e. `a.b.c` instead of `a`), set `leaf_columns = true`
- fn get_schema_by_columns<T>(
- &mut self,
- column_indices: T,
- leaf_columns: bool,
- ) -> Result<Schema>
- where
- T: IntoIterator<Item = usize>;
-
- /// Returns record batch reader from whole parquet file.
- ///
- /// # Arguments
- ///
- /// `batch_size`: The size of each record batch returned from this reader. Only the
- /// last batch may contain records less than this size, otherwise record batches
- /// returned from this reader should contains exactly `batch_size` elements.
- fn get_record_reader(&mut self, batch_size: usize) -> Result<Self::RecordReader>;
-
- /// Returns record batch reader whose record batch contains columns identified by
- /// `column_indices`.
- ///
- /// # Arguments
- ///
- /// `column_indices`: The columns that should be included in record batches.
- /// `batch_size`: Please refer to `get_record_reader`.
- fn get_record_reader_by_columns<T>(
- &mut self,
- column_indices: T,
- batch_size: usize,
- ) -> Result<Self::RecordReader>
- where
- T: IntoIterator<Item = usize>;
-}
-
-pub struct ParquetFileArrowReader {
- file_reader: Arc<dyn FileReader>,
-}
-
-impl ArrowReader for ParquetFileArrowReader {
- type RecordReader = ParquetRecordBatchReader;
-
- fn get_schema(&mut self) -> Result<Schema> {
- let file_metadata = self.file_reader.metadata().file_metadata();
- parquet_to_arrow_schema(
- file_metadata.schema_descr(),
- file_metadata.key_value_metadata(),
- )
- }
-
- fn get_schema_by_columns<T>(
- &mut self,
- column_indices: T,
- leaf_columns: bool,
- ) -> Result<Schema>
- where
- T: IntoIterator<Item = usize>,
- {
- let file_metadata = self.file_reader.metadata().file_metadata();
- if leaf_columns {
- parquet_to_arrow_schema_by_columns(
- file_metadata.schema_descr(),
- column_indices,
- file_metadata.key_value_metadata(),
- )
- } else {
- parquet_to_arrow_schema_by_root_columns(
- file_metadata.schema_descr(),
- column_indices,
- file_metadata.key_value_metadata(),
- )
- }
- }
-
- fn get_record_reader(
- &mut self,
- batch_size: usize,
- ) -> Result<ParquetRecordBatchReader> {
- let column_indices = 0..self
- .file_reader
- .metadata()
- .file_metadata()
- .schema_descr()
- .num_columns();
-
- self.get_record_reader_by_columns(column_indices, batch_size)
- }
-
- fn get_record_reader_by_columns<T>(
- &mut self,
- column_indices: T,
- batch_size: usize,
- ) -> Result<ParquetRecordBatchReader>
- where
- T: IntoIterator<Item = usize>,
- {
- let array_reader = build_array_reader(
- self.file_reader
- .metadata()
- .file_metadata()
- .schema_descr_ptr(),
- self.get_schema()?,
- column_indices,
- self.file_reader.clone(),
- )?;
-
- ParquetRecordBatchReader::try_new(batch_size, array_reader)
- }
-}
-
-impl ParquetFileArrowReader {
- pub fn new(file_reader: Arc<dyn FileReader>) -> Self {
- Self { file_reader }
- }
-
- // Expose the reader metadata
- pub fn get_metadata(&mut self) -> ParquetMetaData {
- self.file_reader.metadata().clone()
- }
-}
-
-pub struct ParquetRecordBatchReader {
- batch_size: usize,
- array_reader: Box<dyn ArrayReader>,
- schema: SchemaRef,
-}
-
-impl Iterator for ParquetRecordBatchReader {
- type Item = ArrowResult<RecordBatch>;
-
- fn next(&mut self) -> Option<Self::Item> {
- match self.array_reader.next_batch(self.batch_size) {
- Err(error) => Some(Err(error.into())),
- Ok(array) => {
- let struct_array =
- array.as_any().downcast_ref::<StructArray>().ok_or_else(|| {
- ArrowError::ParquetError(
- "Struct array reader should return struct array".to_string(),
- )
- });
- match struct_array {
- Err(err) => Some(Err(err)),
- Ok(e) => {
- match RecordBatch::try_new(self.schema.clone(), e.columns_ref()) {
- Err(err) => Some(Err(err)),
- Ok(record_batch) => {
- if record_batch.num_rows() > 0 {
- Some(Ok(record_batch))
- } else {
- None
- }
- }
- }
- }
- }
- }
- }
- }
-}
-
-impl RecordBatchReader for ParquetRecordBatchReader {
- fn schema(&self) -> SchemaRef {
- self.schema.clone()
- }
-}
-
-impl ParquetRecordBatchReader {
- pub fn try_new(
- batch_size: usize,
- array_reader: Box<dyn ArrayReader>,
- ) -> Result<Self> {
- // Check that array reader is struct array reader
- array_reader
- .as_any()
- .downcast_ref::<StructArrayReader>()
- .ok_or_else(|| general_err!("The input must be struct array reader!"))?;
-
- let schema = match array_reader.get_data_type() {
- ArrowType::Struct(ref fields) => Schema::new(fields.clone()),
- _ => unreachable!("Struct array reader's data type is not struct!"),
- };
-
- Ok(Self {
- batch_size,
- array_reader,
- schema: Arc::new(schema),
- })
- }
-}
-
-#[cfg(test)]
-mod tests {
- use crate::arrow::arrow_reader::{ArrowReader, ParquetFileArrowReader};
- use crate::arrow::converter::{
- Converter, FixedSizeArrayConverter, FromConverter, IntervalDayTimeArrayConverter,
- Utf8ArrayConverter,
- };
- use crate::column::writer::get_typed_column_writer_mut;
- use crate::data_type::{
- BoolType, ByteArray, ByteArrayType, DataType, FixedLenByteArray,
- FixedLenByteArrayType, Int32Type,
- };
- use crate::errors::Result;
- use crate::file::properties::WriterProperties;
- use crate::file::reader::{FileReader, SerializedFileReader};
- use crate::file::writer::{FileWriter, SerializedFileWriter};
- use crate::schema::parser::parse_message_type;
- use crate::schema::types::TypePtr;
- use crate::util::test_common::{get_temp_filename, RandGen};
- use arrow::array::*;
- use arrow::record_batch::RecordBatchReader;
- use rand::RngCore;
- use serde_json::json;
- use serde_json::Value::{Array as JArray, Null as JNull, Object as JObject};
- use std::cmp::min;
- use std::convert::TryFrom;
- use std::fs::File;
- use std::path::{Path, PathBuf};
- use std::sync::Arc;
-
- #[test]
- fn test_arrow_reader_all_columns() {
- let json_values = get_json_array("parquet/generated_simple_numerics/blogs.json");
-
- let parquet_file_reader =
- get_test_reader("parquet/generated_simple_numerics/blogs.parquet");
-
- let max_len = parquet_file_reader.metadata().file_metadata().num_rows() as usize;
-
- let mut arrow_reader = ParquetFileArrowReader::new(parquet_file_reader);
-
- let mut record_batch_reader = arrow_reader
- .get_record_reader(60)
- .expect("Failed to read into array!");
-
- // Verify that the schema was correctly parsed
- let original_schema = arrow_reader.get_schema().unwrap().fields().clone();
- assert_eq!(original_schema, *record_batch_reader.schema().fields());
-
- compare_batch_json(&mut record_batch_reader, json_values, max_len);
- }
-
- #[test]
- fn test_arrow_reader_single_column() {
- let json_values = get_json_array("parquet/generated_simple_numerics/blogs.json");
-
- let projected_json_values = json_values
- .into_iter()
- .map(|value| match value {
- JObject(fields) => {
- json!({ "blog_id": fields.get("blog_id").unwrap_or(&JNull).clone()})
- }
- _ => panic!("Input should be json object array!"),
- })
- .collect::<Vec<_>>();
-
- let parquet_file_reader =
- get_test_reader("parquet/generated_simple_numerics/blogs.parquet");
-
- let max_len = parquet_file_reader.metadata().file_metadata().num_rows() as usize;
-
- let mut arrow_reader = ParquetFileArrowReader::new(parquet_file_reader);
-
- let mut record_batch_reader = arrow_reader
- .get_record_reader_by_columns(vec![2], 60)
- .expect("Failed to read into array!");
-
- // Verify that the schema was correctly parsed
- let original_schema = arrow_reader.get_schema().unwrap().fields().clone();
- assert_eq!(1, record_batch_reader.schema().fields().len());
- assert_eq!(original_schema[1], record_batch_reader.schema().fields()[0]);
-
- compare_batch_json(&mut record_batch_reader, projected_json_values, max_len);
- }
-
- #[test]
- fn test_bool_single_column_reader_test() {
- let message_type = "
- message test_schema {
- REQUIRED BOOLEAN leaf;
- }
- ";
-
- let converter = FromConverter::new();
- run_single_column_reader_tests::<
- BoolType,
- BooleanArray,
- FromConverter<Vec<Option<bool>>, BooleanArray>,
- BoolType,
- >(2, message_type, &converter);
- }
-
- struct RandFixedLenGen {}
-
- impl RandGen<FixedLenByteArrayType> for RandFixedLenGen {
- fn gen(len: i32) -> FixedLenByteArray {
- let mut v = vec![0u8; len as usize];
- rand::thread_rng().fill_bytes(&mut v);
- ByteArray::from(v).into()
- }
- }
-
- #[test]
- fn test_fixed_length_binary_column_reader() {
- let message_type = "
- message test_schema {
- REQUIRED FIXED_LEN_BYTE_ARRAY (20) leaf;
- }
- ";
-
- let converter = FixedSizeArrayConverter::new(20);
- run_single_column_reader_tests::<
- FixedLenByteArrayType,
- FixedSizeBinaryArray,
- FixedSizeArrayConverter,
- RandFixedLenGen,
- >(20, message_type, &converter);
- }
-
- #[test]
- fn test_interval_day_time_column_reader() {
- let message_type = "
- message test_schema {
- REQUIRED FIXED_LEN_BYTE_ARRAY (12) leaf (INTERVAL);
- }
- ";
-
- let converter = IntervalDayTimeArrayConverter {};
- run_single_column_reader_tests::<
- FixedLenByteArrayType,
- IntervalDayTimeArray,
- IntervalDayTimeArrayConverter,
- RandFixedLenGen,
- >(12, message_type, &converter);
- }
-
- struct RandUtf8Gen {}
-
- impl RandGen<ByteArrayType> for RandUtf8Gen {
- fn gen(len: i32) -> ByteArray {
- Int32Type::gen(len).to_string().as_str().into()
- }
- }
-
- #[test]
- fn test_utf8_single_column_reader_test() {
- let message_type = "
- message test_schema {
- REQUIRED BINARY leaf (UTF8);
- }
- ";
-
- let converter = Utf8ArrayConverter {};
- run_single_column_reader_tests::<
- ByteArrayType,
- StringArray,
- Utf8ArrayConverter,
- RandUtf8Gen,
- >(2, message_type, &converter);
- }
-
- #[test]
- fn test_read_decimal_file() {
- use arrow::array::DecimalArray;
- let testdata = arrow::util::test_util::parquet_test_data();
- let file_variants = vec![("fixed_length", 25), ("int32", 4), ("int64", 10)];
- for (prefix, target_precision) in file_variants {
- let path = format!("{}/{}_decimal.parquet", testdata, prefix);
- let parquet_reader =
- SerializedFileReader::try_from(File::open(&path).unwrap()).unwrap();
- let mut arrow_reader = ParquetFileArrowReader::new(Arc::new(parquet_reader));
-
- let mut record_reader = arrow_reader.get_record_reader(32).unwrap();
-
- let batch = record_reader.next().unwrap().unwrap();
- assert_eq!(batch.num_rows(), 24);
- let col = batch
- .column(0)
- .as_any()
- .downcast_ref::<DecimalArray>()
- .unwrap();
-
- let expected = 1..25;
-
- assert_eq!(col.precision(), target_precision);
- assert_eq!(col.scale(), 2);
-
- for (i, v) in expected.enumerate() {
- assert_eq!(col.value(i), v * 100_i128);
- }
- }
- }
-
- /// Parameters for single_column_reader_test
- #[derive(Debug)]
- struct TestOptions {
- /// Number of row group to write to parquet (row group size =
- /// num_row_groups / num_rows)
- num_row_groups: usize,
- /// Total number of rows
- num_rows: usize,
- /// Size of batches to read back
- record_batch_size: usize,
- /// Total number of batches to attempt to read.
- /// `record_batch_size` * `num_iterations` should be greater
- /// than `num_rows` to ensure the data can be read back completely
- num_iterations: usize,
- }
-
- /// Create a parquet file and then read it using
- /// `ParquetFileArrowReader` using a standard set of parameters
- /// `opts`.
- ///
- /// `rand_max` represents the maximum size of value to pass to to
- /// value generator
- fn run_single_column_reader_tests<T, A, C, G>(
- rand_max: i32,
- message_type: &str,
- converter: &C,
- ) where
- T: DataType,
- G: RandGen<T>,
- A: PartialEq + Array + 'static,
- C: Converter<Vec<Option<T::T>>, A> + 'static,
- {
- let all_options = vec![
- // choose record_batch_batch (15) so batches cross row
- // group boundaries (50 rows in 2 row groups) cases.
- TestOptions {
- num_row_groups: 2,
- num_rows: 100,
- record_batch_size: 15,
- num_iterations: 50,
- },
- // choose record_batch_batch (5) so batches sometime fall
- // on row group boundaries and (25 rows in 3 row groups
- // --> row groups of 10, 10, and 5). Tests buffer
- // refilling edge cases.
- TestOptions {
- num_row_groups: 3,
- num_rows: 25,
- record_batch_size: 5,
- num_iterations: 50,
- },
- // Choose record_batch_size (25) so all batches fall
- // exactly on row group boundary (25). Tests buffer
- // refilling edge cases.
- TestOptions {
- num_row_groups: 4,
- num_rows: 100,
- record_batch_size: 25,
- num_iterations: 50,
- },
- ];
-
- all_options.into_iter().for_each(|opts| {
- // Print out options to facilitate debugging failures on CI
- println!("Running with Test Options: {:?}", opts);
- single_column_reader_test::<T, A, C, G>(
- opts,
- rand_max,
- message_type,
- converter,
- )
- });
- }
-
- /// Create a parquet file and then read it using
- /// `ParquetFileArrowReader` using the parameters described in
- /// `opts`.
- fn single_column_reader_test<T, A, C, G>(
- opts: TestOptions,
- rand_max: i32,
- message_type: &str,
- converter: &C,
- ) where
- T: DataType,
- G: RandGen<T>,
- A: PartialEq + Array + 'static,
- C: Converter<Vec<Option<T::T>>, A> + 'static,
- {
- let values: Vec<Vec<T::T>> = (0..opts.num_row_groups)
- .map(|_| G::gen_vec(rand_max, opts.num_rows))
- .collect();
-
- let path = get_temp_filename();
-
- let schema = parse_message_type(message_type).map(Arc::new).unwrap();
-
- generate_single_column_file_with_data::<T>(&values, path.as_path(), schema)
- .unwrap();
-
- let parquet_reader =
- SerializedFileReader::try_from(File::open(&path).unwrap()).unwrap();
- let mut arrow_reader = ParquetFileArrowReader::new(Arc::new(parquet_reader));
-
- let mut record_reader = arrow_reader
- .get_record_reader(opts.record_batch_size)
- .unwrap();
-
- let expected_data: Vec<Option<T::T>> = values
- .iter()
- .flat_map(|v| v.iter())
- .map(|b| Some(b.clone()))
- .collect();
-
- for i in 0..opts.num_iterations {
- let start = i * opts.record_batch_size;
-
- let batch = record_reader.next();
- if start < expected_data.len() {
- let end = min(start + opts.record_batch_size, expected_data.len());
- assert!(batch.is_some());
-
- let mut data = vec![];
- data.extend_from_slice(&expected_data[start..end]);
-
- assert_eq!(
- &converter.convert(data).unwrap(),
- batch
- .unwrap()
- .unwrap()
- .column(0)
- .as_any()
- .downcast_ref::<A>()
- .unwrap()
- );
- } else {
- assert!(batch.is_none());
- }
- }
- }
-
- fn generate_single_column_file_with_data<T: DataType>(
- values: &[Vec<T::T>],
- path: &Path,
- schema: TypePtr,
- ) -> Result<parquet_format::FileMetaData> {
- let file = File::create(path)?;
- let writer_props = Arc::new(WriterProperties::builder().build());
-
- let mut writer = SerializedFileWriter::new(file, schema, writer_props)?;
-
- for v in values {
- let mut row_group_writer = writer.next_row_group()?;
- let mut column_writer = row_group_writer
- .next_column()?
- .expect("Column writer is none!");
-
- get_typed_column_writer_mut::<T>(&mut column_writer)
- .write_batch(v, None, None)?;
-
- row_group_writer.close_column(column_writer)?;
- writer.close_row_group(row_group_writer)?
- }
-
- writer.close()
- }
-
- fn get_test_reader(file_name: &str) -> Arc<dyn FileReader> {
- let file = get_test_file(file_name);
-
- let reader =
- SerializedFileReader::new(file).expect("Failed to create serialized reader");
-
- Arc::new(reader)
- }
-
- fn get_test_file(file_name: &str) -> File {
- let mut path = PathBuf::new();
- path.push(arrow::util::test_util::arrow_test_data());
- path.push(file_name);
-
- File::open(path.as_path()).expect("File not found!")
- }
-
- fn get_json_array(filename: &str) -> Vec<serde_json::Value> {
- match serde_json::from_reader(get_test_file(filename))
- .expect("Failed to read json value from file!")
- {
- JArray(values) => values,
- _ => panic!("Input should be json array!"),
- }
- }
-
- fn compare_batch_json(
- record_batch_reader: &mut dyn RecordBatchReader,
- json_values: Vec<serde_json::Value>,
- max_len: usize,
- ) {
- for i in 0..20 {
- let array: Option<StructArray> = record_batch_reader
- .next()
- .map(|r| r.expect("Failed to read record batch!").into());
-
- let (start, end) = (i * 60_usize, (i + 1) * 60_usize);
-
- if start < max_len {
- assert!(array.is_some());
- assert_ne!(0, array.as_ref().unwrap().len());
- let end = min(end, max_len);
- let json = JArray(Vec::from(&json_values[start..end]));
- assert_eq!(array.unwrap(), json)
- } else {
- assert!(array.is_none());
- }
- }
- }
-
- #[test]
- fn test_read_structs() {
- // This particular test file has columns of struct types where there is
- // a column that has the same name as one of the struct fields
- // (see: ARROW-11452)
- let testdata = arrow::util::test_util::parquet_test_data();
- let path = format!("{}/nested_structs.rust.parquet", testdata);
- let parquet_file_reader =
- SerializedFileReader::try_from(File::open(&path).unwrap()).unwrap();
- let mut arrow_reader = ParquetFileArrowReader::new(Arc::new(parquet_file_reader));
- let record_batch_reader = arrow_reader
- .get_record_reader(60)
- .expect("Failed to read into array!");
-
- for batch in record_batch_reader {
- batch.unwrap();
- }
- }
-}
diff --git a/parquet/src/arrow/arrow_writer.rs b/parquet/src/arrow/arrow_writer.rs
deleted file mode 100644
index 69ebce6..0000000
--- a/parquet/src/arrow/arrow_writer.rs
+++ /dev/null
@@ -1,1678 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-//! Contains writer which writes arrow data into parquet data.
-
-use std::sync::Arc;
-
-use arrow::array as arrow_array;
-use arrow::datatypes::{DataType as ArrowDataType, IntervalUnit, SchemaRef};
-use arrow::record_batch::RecordBatch;
-use arrow_array::Array;
-
-use super::levels::LevelInfo;
-use super::schema::{
- add_encoded_arrow_schema_to_metadata, decimal_length_from_precision,
-};
-
-use crate::column::writer::ColumnWriter;
-use crate::errors::{ParquetError, Result};
-use crate::file::properties::WriterProperties;
-use crate::{
- data_type::*,
- file::writer::{FileWriter, ParquetWriter, RowGroupWriter, SerializedFileWriter},
-};
-
-/// Arrow writer
-///
-/// Writes Arrow `RecordBatch`es to a Parquet writer
-pub struct ArrowWriter<W: ParquetWriter> {
- /// Underlying Parquet writer
- writer: SerializedFileWriter<W>,
- /// A copy of the Arrow schema.
- ///
- /// The schema is used to verify that each record batch written has the correct schema
- arrow_schema: SchemaRef,
- /// The length of arrays to write to each row group
- max_row_group_size: usize,
-}
-
-impl<W: 'static + ParquetWriter> ArrowWriter<W> {
- /// Try to create a new Arrow writer
- ///
- /// The writer will fail if:
- /// * a `SerializedFileWriter` cannot be created from the ParquetWriter
- /// * the Arrow schema contains unsupported datatypes such as Unions
- pub fn try_new(
- writer: W,
- arrow_schema: SchemaRef,
- props: Option<WriterProperties>,
- ) -> Result<Self> {
- let schema = crate::arrow::arrow_to_parquet_schema(&arrow_schema)?;
- // add serialized arrow schema
- let mut props = props.unwrap_or_else(|| WriterProperties::builder().build());
- add_encoded_arrow_schema_to_metadata(&arrow_schema, &mut props);
-
- let max_row_group_size = props.max_row_group_size();
-
- let file_writer = SerializedFileWriter::new(
- writer.try_clone()?,
- schema.root_schema_ptr(),
- Arc::new(props),
- )?;
-
- Ok(Self {
- writer: file_writer,
- arrow_schema,
- max_row_group_size,
- })
- }
-
- /// Write a RecordBatch to writer
- ///
- /// The writer will slice the `batch` into `max_row_group_size`,
- /// but if a batch has left-over rows less than the row group size,
- /// the last row group will have fewer records.
- /// This is currently a limitation because we close the row group
- /// instead of keeping it open for the next batch.
- pub fn write(&mut self, batch: &RecordBatch) -> Result<()> {
- // validate batch schema against writer's supplied schema
- if self.arrow_schema != batch.schema() {
- return Err(ParquetError::ArrowError(
- "Record batch schema does not match writer schema".to_string(),
- ));
- }
- // Track the number of rows being written in the batch.
- // We currently do not have a way of slicing nested arrays, thus we
- // track this manually.
- let num_rows = batch.num_rows();
- let batches = (num_rows + self.max_row_group_size - 1) / self.max_row_group_size;
- let min_batch = num_rows.min(self.max_row_group_size);
- for batch_index in 0..batches {
- // Determine the offset and length of arrays
- let offset = batch_index * min_batch;
- let length = (num_rows - offset).min(self.max_row_group_size);
-
- // Compute the definition and repetition levels of the batch
- let batch_level = LevelInfo::new(offset, length);
- let mut row_group_writer = self.writer.next_row_group()?;
- for (array, field) in batch.columns().iter().zip(batch.schema().fields()) {
- let mut levels = batch_level.calculate_array_levels(array, field);
- // Reverse levels as we pop() them when writing arrays
- levels.reverse();
- write_leaves(&mut row_group_writer, array, &mut levels)?;
- }
-
- self.writer.close_row_group(row_group_writer)?;
- }
-
- Ok(())
- }
-
- /// Close and finalize the underlying Parquet writer
- pub fn close(&mut self) -> Result<parquet_format::FileMetaData> {
- self.writer.close()
- }
-}
-
-/// Convenience method to get the next ColumnWriter from the RowGroupWriter
-#[inline]
-#[allow(clippy::borrowed_box)]
-fn get_col_writer(
- row_group_writer: &mut Box<dyn RowGroupWriter>,
-) -> Result<ColumnWriter> {
- let col_writer = row_group_writer
- .next_column()?
- .expect("Unable to get column writer");
- Ok(col_writer)
-}
-
-#[allow(clippy::borrowed_box)]
-fn write_leaves(
- mut row_group_writer: &mut Box<dyn RowGroupWriter>,
- array: &arrow_array::ArrayRef,
- mut levels: &mut Vec<LevelInfo>,
-) -> Result<()> {
- match array.data_type() {
- ArrowDataType::Null
- | ArrowDataType::Boolean
- | ArrowDataType::Int8
- | ArrowDataType::Int16
- | ArrowDataType::Int32
- | ArrowDataType::Int64
- | ArrowDataType::UInt8
- | ArrowDataType::UInt16
- | ArrowDataType::UInt32
- | ArrowDataType::UInt64
- | ArrowDataType::Float32
- | ArrowDataType::Float64
- | ArrowDataType::Timestamp(_, _)
- | ArrowDataType::Date32
- | ArrowDataType::Date64
- | ArrowDataType::Time32(_)
- | ArrowDataType::Time64(_)
- | ArrowDataType::Duration(_)
- | ArrowDataType::Interval(_)
- | ArrowDataType::LargeBinary
- | ArrowDataType::Binary
- | ArrowDataType::Utf8
- | ArrowDataType::LargeUtf8
- | ArrowDataType::Decimal(_, _)
- | ArrowDataType::FixedSizeBinary(_) => {
- let mut col_writer = get_col_writer(&mut row_group_writer)?;
- write_leaf(
- &mut col_writer,
- array,
- levels.pop().expect("Levels exhausted"),
- )?;
- row_group_writer.close_column(col_writer)?;
- Ok(())
- }
- ArrowDataType::List(_) | ArrowDataType::LargeList(_) => {
- // write the child list
- let data = array.data();
- let child_array = arrow_array::make_array(data.child_data()[0].clone());
- write_leaves(&mut row_group_writer, &child_array, &mut levels)?;
- Ok(())
- }
- ArrowDataType::Struct(_) => {
- let struct_array: &arrow_array::StructArray = array
- .as_any()
- .downcast_ref::<arrow_array::StructArray>()
- .expect("Unable to get struct array");
- for field in struct_array.columns() {
- write_leaves(&mut row_group_writer, field, &mut levels)?;
- }
- Ok(())
- }
- ArrowDataType::Dictionary(_, value_type) => {
- // cast dictionary to a primitive
- let array = arrow::compute::cast(array, value_type)?;
-
- let mut col_writer = get_col_writer(&mut row_group_writer)?;
- write_leaf(
- &mut col_writer,
- &array,
- levels.pop().expect("Levels exhausted"),
- )?;
- row_group_writer.close_column(col_writer)?;
- Ok(())
- }
- ArrowDataType::Float16 => Err(ParquetError::ArrowError(
- "Float16 arrays not supported".to_string(),
- )),
- ArrowDataType::FixedSizeList(_, _) | ArrowDataType::Union(_) => {
- Err(ParquetError::NYI(
- format!(
- "Attempting to write an Arrow type {:?} to parquet that is not yet implemented",
- array.data_type()
- )
- ))
- }
- }
-}
-
-fn write_leaf(
- writer: &mut ColumnWriter,
- column: &arrow_array::ArrayRef,
- levels: LevelInfo,
-) -> Result<i64> {
- let indices = levels.filter_array_indices();
- // Slice array according to computed offset and length
- let column = column.slice(levels.offset, levels.length);
- let written = match writer {
- ColumnWriter::Int32ColumnWriter(ref mut typed) => {
- let values = match column.data_type() {
- ArrowDataType::Date64 => {
- // If the column is a Date64, we cast it to a Date32, and then interpret that as Int32
- let array = if let ArrowDataType::Date64 = column.data_type() {
- let array =
- arrow::compute::cast(&column, &ArrowDataType::Date32)?;
- arrow::compute::cast(&array, &ArrowDataType::Int32)?
- } else {
- arrow::compute::cast(&column, &ArrowDataType::Int32)?
- };
- let array = array
- .as_any()
- .downcast_ref::<arrow_array::Int32Array>()
- .expect("Unable to get int32 array");
- get_numeric_array_slice::<Int32Type, _>(&array, &indices)
- }
- ArrowDataType::UInt32 => {
- // follow C++ implementation and use overflow/reinterpret cast from u32 to i32 which will map
- // `(i32::MAX as u32)..u32::MAX` to `i32::MIN..0`
- let array = column
- .as_any()
- .downcast_ref::<arrow_array::UInt32Array>()
- .expect("Unable to get u32 array");
- let array = arrow::compute::unary::<_, _, arrow::datatypes::Int32Type>(
- array,
- |x| x as i32,
- );
- get_numeric_array_slice::<Int32Type, _>(&array, &indices)
- }
- _ => {
- let array = arrow::compute::cast(&column, &ArrowDataType::Int32)?;
- let array = array
- .as_any()
- .downcast_ref::<arrow_array::Int32Array>()
- .expect("Unable to get i32 array");
- get_numeric_array_slice::<Int32Type, _>(&array, &indices)
- }
- };
- typed.write_batch(
- values.as_slice(),
- Some(levels.definition.as_slice()),
- levels.repetition.as_deref(),
- )?
- }
- ColumnWriter::BoolColumnWriter(ref mut typed) => {
- let array = column
- .as_any()
- .downcast_ref::<arrow_array::BooleanArray>()
- .expect("Unable to get boolean array");
- typed.write_batch(
- get_bool_array_slice(&array, &indices).as_slice(),
- Some(levels.definition.as_slice()),
- levels.repetition.as_deref(),
- )?
- }
- ColumnWriter::Int64ColumnWriter(ref mut typed) => {
- let values = match column.data_type() {
- ArrowDataType::Int64 => {
- let array = column
- .as_any()
- .downcast_ref::<arrow_array::Int64Array>()
- .expect("Unable to get i64 array");
- get_numeric_array_slice::<Int64Type, _>(&array, &indices)
- }
- ArrowDataType::UInt64 => {
- // follow C++ implementation and use overflow/reinterpret cast from u64 to i64 which will map
- // `(i64::MAX as u64)..u64::MAX` to `i64::MIN..0`
- let array = column
- .as_any()
- .downcast_ref::<arrow_array::UInt64Array>()
- .expect("Unable to get u64 array");
- let array = arrow::compute::unary::<_, _, arrow::datatypes::Int64Type>(
- array,
- |x| x as i64,
- );
- get_numeric_array_slice::<Int64Type, _>(&array, &indices)
- }
- _ => {
- let array = arrow::compute::cast(&column, &ArrowDataType::Int64)?;
- let array = array
- .as_any()
- .downcast_ref::<arrow_array::Int64Array>()
- .expect("Unable to get i64 array");
- get_numeric_array_slice::<Int64Type, _>(&array, &indices)
- }
- };
- typed.write_batch(
- values.as_slice(),
- Some(levels.definition.as_slice()),
- levels.repetition.as_deref(),
- )?
- }
- ColumnWriter::Int96ColumnWriter(ref mut _typed) => {
- unreachable!("Currently unreachable because data type not supported")
- }
- ColumnWriter::FloatColumnWriter(ref mut typed) => {
- let array = column
- .as_any()
- .downcast_ref::<arrow_array::Float32Array>()
- .expect("Unable to get Float32 array");
- typed.write_batch(
- get_numeric_array_slice::<FloatType, _>(&array, &indices).as_slice(),
- Some(levels.definition.as_slice()),
- levels.repetition.as_deref(),
- )?
- }
- ColumnWriter::DoubleColumnWriter(ref mut typed) => {
- let array = column
- .as_any()
- .downcast_ref::<arrow_array::Float64Array>()
- .expect("Unable to get Float64 array");
- typed.write_batch(
- get_numeric_array_slice::<DoubleType, _>(&array, &indices).as_slice(),
- Some(levels.definition.as_slice()),
- levels.repetition.as_deref(),
- )?
- }
- ColumnWriter::ByteArrayColumnWriter(ref mut typed) => match column.data_type() {
- ArrowDataType::Binary => {
- let array = column
- .as_any()
- .downcast_ref::<arrow_array::BinaryArray>()
- .expect("Unable to get BinaryArray array");
- typed.write_batch(
- get_binary_array(&array).as_slice(),
- Some(levels.definition.as_slice()),
- levels.repetition.as_deref(),
- )?
- }
- ArrowDataType::Utf8 => {
- let array = column
- .as_any()
- .downcast_ref::<arrow_array::StringArray>()
- .expect("Unable to get LargeBinaryArray array");
- typed.write_batch(
- get_string_array(&array).as_slice(),
- Some(levels.definition.as_slice()),
- levels.repetition.as_deref(),
- )?
- }
- ArrowDataType::LargeBinary => {
- let array = column
- .as_any()
- .downcast_ref::<arrow_array::LargeBinaryArray>()
- .expect("Unable to get LargeBinaryArray array");
- typed.write_batch(
- get_large_binary_array(&array).as_slice(),
- Some(levels.definition.as_slice()),
- levels.repetition.as_deref(),
- )?
- }
- ArrowDataType::LargeUtf8 => {
- let array = column
- .as_any()
- .downcast_ref::<arrow_array::LargeStringArray>()
- .expect("Unable to get LargeUtf8 array");
- typed.write_batch(
- get_large_string_array(&array).as_slice(),
- Some(levels.definition.as_slice()),
- levels.repetition.as_deref(),
- )?
- }
- _ => unreachable!("Currently unreachable because data type not supported"),
- },
- ColumnWriter::FixedLenByteArrayColumnWriter(ref mut typed) => {
- let bytes = match column.data_type() {
- ArrowDataType::Interval(interval_unit) => match interval_unit {
- IntervalUnit::YearMonth => {
- let array = column
- .as_any()
- .downcast_ref::<arrow_array::IntervalYearMonthArray>()
- .unwrap();
- get_interval_ym_array_slice(&array, &indices)
- }
- IntervalUnit::DayTime => {
- let array = column
- .as_any()
- .downcast_ref::<arrow_array::IntervalDayTimeArray>()
- .unwrap();
- get_interval_dt_array_slice(&array, &indices)
- }
- },
- ArrowDataType::FixedSizeBinary(_) => {
- let array = column
- .as_any()
- .downcast_ref::<arrow_array::FixedSizeBinaryArray>()
- .unwrap();
- get_fsb_array_slice(&array, &indices)
- }
- ArrowDataType::Decimal(_, _) => {
- let array = column
- .as_any()
- .downcast_ref::<arrow_array::DecimalArray>()
- .unwrap();
- get_decimal_array_slice(&array, &indices)
- }
- _ => {
- return Err(ParquetError::NYI(
- "Attempting to write an Arrow type that is not yet implemented"
- .to_string(),
- ));
- }
- };
- typed.write_batch(
- bytes.as_slice(),
- Some(levels.definition.as_slice()),
- levels.repetition.as_deref(),
- )?
- }
- };
- Ok(written as i64)
-}
-
-macro_rules! def_get_binary_array_fn {
- ($name:ident, $ty:ty) => {
- fn $name(array: &$ty) -> Vec<ByteArray> {
- let mut values = Vec::with_capacity(array.len() - array.null_count());
- for i in 0..array.len() {
- if array.is_valid(i) {
- let bytes: Vec<u8> = array.value(i).into();
- let bytes = ByteArray::from(bytes);
- values.push(bytes);
- }
- }
- values
- }
- };
-}
-
-def_get_binary_array_fn!(get_binary_array, arrow_array::BinaryArray);
-def_get_binary_array_fn!(get_string_array, arrow_array::StringArray);
-def_get_binary_array_fn!(get_large_binary_array, arrow_array::LargeBinaryArray);
-def_get_binary_array_fn!(get_large_string_array, arrow_array::LargeStringArray);
-
-/// Get the underlying numeric array slice, skipping any null values.
-/// If there are no null values, it might be quicker to get the slice directly instead of
-/// calling this function.
-fn get_numeric_array_slice<T, A>(
- array: &arrow_array::PrimitiveArray<A>,
- indices: &[usize],
-) -> Vec<T::T>
-where
- T: DataType,
- A: arrow::datatypes::ArrowNumericType,
- T::T: From<A::Native>,
-{
- let mut values = Vec::with_capacity(indices.len());
- for i in indices {
- values.push(array.value(*i).into())
- }
- values
-}
-
-fn get_bool_array_slice(
- array: &arrow_array::BooleanArray,
- indices: &[usize],
-) -> Vec<bool> {
- let mut values = Vec::with_capacity(indices.len());
- for i in indices {
- values.push(array.value(*i))
- }
- values
-}
-
-/// Returns 12-byte values representing 3 values of months, days and milliseconds (4-bytes each).
-/// An Arrow YearMonth interval only stores months, thus only the first 4 bytes are populated.
-fn get_interval_ym_array_slice(
- array: &arrow_array::IntervalYearMonthArray,
- indices: &[usize],
-) -> Vec<FixedLenByteArray> {
- let mut values = Vec::with_capacity(indices.len());
- for i in indices {
- let mut value = array.value(*i).to_le_bytes().to_vec();
- let mut suffix = vec![0; 8];
- value.append(&mut suffix);
- values.push(FixedLenByteArray::from(ByteArray::from(value)))
- }
- values
-}
-
-/// Returns 12-byte values representing 3 values of months, days and milliseconds (4-bytes each).
-/// An Arrow DayTime interval only stores days and millis, thus the first 4 bytes are not populated.
-fn get_interval_dt_array_slice(
- array: &arrow_array::IntervalDayTimeArray,
- indices: &[usize],
-) -> Vec<FixedLenByteArray> {
- let mut values = Vec::with_capacity(indices.len());
- for i in indices {
- let mut prefix = vec![0; 4];
- let mut value = array.value(*i).to_le_bytes().to_vec();
- prefix.append(&mut value);
- debug_assert_eq!(prefix.len(), 12);
- values.push(FixedLenByteArray::from(ByteArray::from(prefix)));
- }
- values
-}
-
-fn get_decimal_array_slice(
- array: &arrow_array::DecimalArray,
- indices: &[usize],
-) -> Vec<FixedLenByteArray> {
- let mut values = Vec::with_capacity(indices.len());
- let size = decimal_length_from_precision(array.precision());
- for i in indices {
- let as_be_bytes = array.value(*i).to_be_bytes();
- let resized_value = as_be_bytes[(16 - size)..].to_vec();
- values.push(FixedLenByteArray::from(ByteArray::from(resized_value)));
- }
- values
-}
-
-fn get_fsb_array_slice(
- array: &arrow_array::FixedSizeBinaryArray,
- indices: &[usize],
-) -> Vec<FixedLenByteArray> {
- let mut values = Vec::with_capacity(indices.len());
- for i in indices {
- let value = array.value(*i).to_vec();
- values.push(FixedLenByteArray::from(ByteArray::from(value)))
- }
- values
-}
-
-#[cfg(test)]
-mod tests {
- use super::*;
-
- use std::sync::Arc;
- use std::{fs::File, io::Seek};
-
- use arrow::datatypes::ToByteSlice;
- use arrow::datatypes::{DataType, Field, Schema, UInt32Type, UInt8Type};
- use arrow::record_batch::RecordBatch;
- use arrow::{array::*, buffer::Buffer};
-
- use crate::arrow::{ArrowReader, ParquetFileArrowReader};
- use crate::file::{
- reader::{FileReader, SerializedFileReader},
- statistics::Statistics,
- writer::InMemoryWriteableCursor,
- };
- use crate::util::test_common::get_temp_file;
-
- #[test]
- fn arrow_writer() {
- // define schema
- let schema = Schema::new(vec![
- Field::new("a", DataType::Int32, false),
- Field::new("b", DataType::Int32, true),
- ]);
-
- // create some data
- let a = Int32Array::from(vec![1, 2, 3, 4, 5]);
- let b = Int32Array::from(vec![Some(1), None, None, Some(4), Some(5)]);
-
- // build a record batch
- let batch = RecordBatch::try_new(
- Arc::new(schema.clone()),
- vec![Arc::new(a), Arc::new(b)],
- )
- .unwrap();
-
- let file = get_temp_file("test_arrow_writer.parquet", &[]);
- let mut writer = ArrowWriter::try_new(file, Arc::new(schema), None).unwrap();
- writer.write(&batch).unwrap();
- writer.close().unwrap();
- }
-
- #[test]
- fn roundtrip_bytes() {
- // define schema
- let schema = Arc::new(Schema::new(vec![
- Field::new("a", DataType::Int32, false),
- Field::new("b", DataType::Int32, true),
- ]));
-
- // create some data
- let a = Int32Array::from(vec![1, 2, 3, 4, 5]);
- let b = Int32Array::from(vec![Some(1), None, None, Some(4), Some(5)]);
-
- // build a record batch
- let expected_batch =
- RecordBatch::try_new(schema.clone(), vec![Arc::new(a), Arc::new(b)]).unwrap();
-
- let cursor = InMemoryWriteableCursor::default();
-
- {
- let mut writer = ArrowWriter::try_new(cursor.clone(), schema, None).unwrap();
- writer.write(&expected_batch).unwrap();
- writer.close().unwrap();
- }
-
- let buffer = cursor.into_inner().unwrap();
-
- let cursor = crate::file::serialized_reader::SliceableCursor::new(buffer);
- let reader = SerializedFileReader::new(cursor).unwrap();
- let mut arrow_reader = ParquetFileArrowReader::new(Arc::new(reader));
- let mut record_batch_reader = arrow_reader.get_record_reader(1024).unwrap();
-
- let actual_batch = record_batch_reader
- .next()
- .expect("No batch found")
- .expect("Unable to get batch");
-
- assert_eq!(expected_batch.schema(), actual_batch.schema());
- assert_eq!(expected_batch.num_columns(), actual_batch.num_columns());
- assert_eq!(expected_batch.num_rows(), actual_batch.num_rows());
- for i in 0..expected_batch.num_columns() {
- let expected_data = expected_batch.column(i).data().clone();
- let actual_data = actual_batch.column(i).data().clone();
-
- assert_eq!(expected_data, actual_data);
- }
- }
-
- #[test]
- fn arrow_writer_non_null() {
- // define schema
- let schema = Schema::new(vec![Field::new("a", DataType::Int32, false)]);
-
- // create some data
- let a = Int32Array::from(vec![1, 2, 3, 4, 5]);
-
- // build a record batch
- let batch =
- RecordBatch::try_new(Arc::new(schema.clone()), vec![Arc::new(a)]).unwrap();
-
- let file = get_temp_file("test_arrow_writer_non_null.parquet", &[]);
- let mut writer = ArrowWriter::try_new(file, Arc::new(schema), None).unwrap();
- writer.write(&batch).unwrap();
- writer.close().unwrap();
- }
-
- #[test]
- fn arrow_writer_list() {
- // define schema
- let schema = Schema::new(vec![Field::new(
- "a",
- DataType::List(Box::new(Field::new("item", DataType::Int32, true))),
- false,
- )]);
-
- // create some data
- let a_values = Int32Array::from(vec![1, 2, 3, 4, 5, 6, 7, 8, 9, 10]);
-
- // Construct a buffer for value offsets, for the nested array:
- // [[1], [2, 3], null, [4, 5, 6], [7, 8, 9, 10]]
- let a_value_offsets =
- arrow::buffer::Buffer::from(&[0, 1, 3, 3, 6, 10].to_byte_slice());
-
- // Construct a list array from the above two
- let a_list_data = ArrayData::builder(DataType::List(Box::new(Field::new(
- "item",
- DataType::Int32,
- true,
- ))))
- .len(5)
- .add_buffer(a_value_offsets)
- .add_child_data(a_values.data().clone())
- .null_bit_buffer(Buffer::from(vec![0b00011011]))
- .build();
- let a = ListArray::from(a_list_data);
-
- // build a record batch
- let batch =
- RecordBatch::try_new(Arc::new(schema.clone()), vec![Arc::new(a)]).unwrap();
-
- // I think this setup is incorrect because this should pass
- assert_eq!(batch.column(0).data().null_count(), 1);
-
- let file = get_temp_file("test_arrow_writer_list.parquet", &[]);
- let mut writer = ArrowWriter::try_new(file, Arc::new(schema), None).unwrap();
- writer.write(&batch).unwrap();
- writer.close().unwrap();
- }
-
- #[test]
- fn arrow_writer_binary() {
- let string_field = Field::new("a", DataType::Utf8, false);
- let binary_field = Field::new("b", DataType::Binary, false);
- let schema = Schema::new(vec![string_field, binary_field]);
-
- let raw_string_values = vec!["foo", "bar", "baz", "quux"];
- let raw_binary_values = vec![
- b"foo".to_vec(),
- b"bar".to_vec(),
- b"baz".to_vec(),
- b"quux".to_vec(),
- ];
- let raw_binary_value_refs = raw_binary_values
- .iter()
- .map(|x| x.as_slice())
- .collect::<Vec<_>>();
-
- let string_values = StringArray::from(raw_string_values.clone());
- let binary_values = BinaryArray::from(raw_binary_value_refs);
- let batch = RecordBatch::try_new(
- Arc::new(schema.clone()),
- vec![Arc::new(string_values), Arc::new(binary_values)],
- )
- .unwrap();
-
- let mut file = get_temp_file("test_arrow_writer_binary.parquet", &[]);
- let mut writer =
- ArrowWriter::try_new(file.try_clone().unwrap(), Arc::new(schema), None)
- .unwrap();
- writer.write(&batch).unwrap();
- writer.close().unwrap();
-
- file.seek(std::io::SeekFrom::Start(0)).unwrap();
- let file_reader = SerializedFileReader::new(file).unwrap();
- let mut arrow_reader = ParquetFileArrowReader::new(Arc::new(file_reader));
- let mut record_batch_reader = arrow_reader.get_record_reader(1024).unwrap();
-
- let batch = record_batch_reader.next().unwrap().unwrap();
- let string_col = batch
- .column(0)
- .as_any()
- .downcast_ref::<StringArray>()
- .unwrap();
- let binary_col = batch
- .column(1)
- .as_any()
- .downcast_ref::<BinaryArray>()
- .unwrap();
-
- for i in 0..batch.num_rows() {
- assert_eq!(string_col.value(i), raw_string_values[i]);
- assert_eq!(binary_col.value(i), raw_binary_values[i].as_slice());
- }
- }
-
- #[test]
- fn arrow_writer_decimal() {
- let decimal_field = Field::new("a", DataType::Decimal(5, 2), false);
- let schema = Schema::new(vec![decimal_field]);
-
- let mut dec_builder = DecimalBuilder::new(4, 5, 2);
- dec_builder.append_value(10_000).unwrap();
- dec_builder.append_value(50_000).unwrap();
- dec_builder.append_value(0).unwrap();
- dec_builder.append_value(-100).unwrap();
-
- let raw_decimal_i128_values: Vec<i128> = vec![10_000, 50_000, 0, -100];
- let decimal_values = dec_builder.finish();
- let batch = RecordBatch::try_new(
- Arc::new(schema.clone()),
- vec![Arc::new(decimal_values)],
- )
- .unwrap();
-
- let mut file = get_temp_file("test_arrow_writer_decimal.parquet", &[]);
- let mut writer =
- ArrowWriter::try_new(file.try_clone().unwrap(), Arc::new(schema), None)
- .unwrap();
- writer.write(&batch).unwrap();
- writer.close().unwrap();
-
- file.seek(std::io::SeekFrom::Start(0)).unwrap();
- let file_reader = SerializedFileReader::new(file).unwrap();
- let mut arrow_reader = ParquetFileArrowReader::new(Arc::new(file_reader));
- let mut record_batch_reader = arrow_reader.get_record_reader(1024).unwrap();
-
- let batch = record_batch_reader.next().unwrap().unwrap();
- let decimal_col = batch
- .column(0)
- .as_any()
- .downcast_ref::<DecimalArray>()
- .unwrap();
-
- for i in 0..batch.num_rows() {
- assert_eq!(decimal_col.value(i), raw_decimal_i128_values[i]);
- }
- }
-
- #[test]
- fn arrow_writer_complex() {
- // define schema
- let struct_field_d = Field::new("d", DataType::Float64, true);
- let struct_field_f = Field::new("f", DataType::Float32, true);
- let struct_field_g = Field::new(
- "g",
- DataType::List(Box::new(Field::new("item", DataType::Int16, true))),
- false,
- );
- let struct_field_h = Field::new(
- "h",
- DataType::List(Box::new(Field::new("item", DataType::Int16, false))),
- true,
- );
- let struct_field_e = Field::new(
- "e",
- DataType::Struct(vec![
- struct_field_f.clone(),
- struct_field_g.clone(),
- struct_field_h.clone(),
- ]),
- false,
- );
- let schema = Schema::new(vec![
- Field::new("a", DataType::Int32, false),
- Field::new("b", DataType::Int32, true),
- Field::new(
- "c",
- DataType::Struct(vec![struct_field_d.clone(), struct_field_e.clone()]),
- false,
- ),
- ]);
-
- // create some data
- let a = Int32Array::from(vec![1, 2, 3, 4, 5]);
- let b = Int32Array::from(vec![Some(1), None, None, Some(4), Some(5)]);
- let d = Float64Array::from(vec![None, None, None, Some(1.0), None]);
- let f = Float32Array::from(vec![Some(0.0), None, Some(333.3), None, Some(5.25)]);
-
- let g_value = Int16Array::from(vec![1, 2, 3, 4, 5, 6, 7, 8, 9, 10]);
-
- // Construct a buffer for value offsets, for the nested array:
- // [[1], [2, 3], [], [4, 5, 6], [7, 8, 9, 10]]
- let g_value_offsets =
- arrow::buffer::Buffer::from(&[0, 1, 3, 3, 6, 10].to_byte_slice());
-
- // Construct a list array from the above two
- let g_list_data = ArrayData::builder(struct_field_g.data_type().clone())
- .len(5)
- .add_buffer(g_value_offsets.clone())
- .add_child_data(g_value.data().clone())
- .build();
- let g = ListArray::from(g_list_data);
- // The difference between g and h is that h has a null bitmap
- let h_list_data = ArrayData::builder(struct_field_h.data_type().clone())
- .len(5)
- .add_buffer(g_value_offsets)
- .add_child_data(g_value.data().clone())
- .null_bit_buffer(Buffer::from(vec![0b00011011]))
- .build();
- let h = ListArray::from(h_list_data);
-
- let e = StructArray::from(vec![
- (struct_field_f, Arc::new(f) as ArrayRef),
- (struct_field_g, Arc::new(g) as ArrayRef),
- (struct_field_h, Arc::new(h) as ArrayRef),
- ]);
-
- let c = StructArray::from(vec![
- (struct_field_d, Arc::new(d) as ArrayRef),
- (struct_field_e, Arc::new(e) as ArrayRef),
- ]);
-
- // build a record batch
- let batch = RecordBatch::try_new(
- Arc::new(schema),
- vec![Arc::new(a), Arc::new(b), Arc::new(c)],
- )
- .unwrap();
-
- roundtrip(
- "test_arrow_writer_complex.parquet",
- batch.clone(),
- Some(SMALL_SIZE / 2),
- );
-
- roundtrip(
- "test_arrow_writer_complex_small_batch.parquet",
- batch,
- Some(SMALL_SIZE / 3),
- );
- }
-
- #[test]
- fn arrow_writer_complex_mixed() {
- // This test was added while investigating https://github.com/apache/arrow-rs/issues/244.
- // It was subsequently fixed while investigating https://github.com/apache/arrow-rs/issues/245.
-
- // define schema
- let offset_field = Field::new("offset", DataType::Int32, false);
- let partition_field = Field::new("partition", DataType::Int64, true);
- let topic_field = Field::new("topic", DataType::Utf8, true);
- let schema = Schema::new(vec![Field::new(
- "some_nested_object",
- DataType::Struct(vec![
- offset_field.clone(),
- partition_field.clone(),
- topic_field.clone(),
- ]),
- false,
- )]);
-
- // create some data
- let offset = Int32Array::from(vec![1, 2, 3, 4, 5]);
- let partition = Int64Array::from(vec![Some(1), None, None, Some(4), Some(5)]);
- let topic = StringArray::from(vec![Some("A"), None, Some("A"), Some(""), None]);
-
- let some_nested_object = StructArray::from(vec![
- (offset_field, Arc::new(offset) as ArrayRef),
- (partition_field, Arc::new(partition) as ArrayRef),
- (topic_field, Arc::new(topic) as ArrayRef),
- ]);
-
- // build a record batch
- let batch =
- RecordBatch::try_new(Arc::new(schema), vec![Arc::new(some_nested_object)])
- .unwrap();
-
- roundtrip(
- "test_arrow_writer_complex_mixed.parquet",
- batch,
- Some(SMALL_SIZE / 2),
- );
- }
-
- #[test]
- fn arrow_writer_2_level_struct() {
- // tests writing <struct<struct<primitive>>
- let field_c = Field::new("c", DataType::Int32, true);
- let field_b = Field::new("b", DataType::Struct(vec![field_c]), true);
- let field_a = Field::new("a", DataType::Struct(vec![field_b.clone()]), true);
- let schema = Schema::new(vec![field_a.clone()]);
-
- // create data
- let c = Int32Array::from(vec![Some(1), None, Some(3), None, None, Some(6)]);
- let b_data = ArrayDataBuilder::new(field_b.data_type().clone())
- .len(6)
- .null_bit_buffer(Buffer::from(vec![0b00100111]))
- .add_child_data(c.data().clone())
- .build();
- let b = StructArray::from(b_data);
- let a_data = ArrayDataBuilder::new(field_a.data_type().clone())
- .len(6)
- .null_bit_buffer(Buffer::from(vec![0b00101111]))
- .add_child_data(b.data().clone())
- .build();
- let a = StructArray::from(a_data);
-
- assert_eq!(a.null_count(), 1);
- assert_eq!(a.column(0).null_count(), 2);
-
- // build a racord batch
- let batch = RecordBatch::try_new(Arc::new(schema), vec![Arc::new(a)]).unwrap();
-
- roundtrip(
- "test_arrow_writer_2_level_struct.parquet",
- batch,
- Some(SMALL_SIZE / 2),
- );
- }
-
- #[test]
- fn arrow_writer_2_level_struct_non_null() {
- // tests writing <struct<struct<primitive>>
- let field_c = Field::new("c", DataType::Int32, false);
- let field_b = Field::new("b", DataType::Struct(vec![field_c]), false);
- let field_a = Field::new("a", DataType::Struct(vec![field_b.clone()]), false);
- let schema = Schema::new(vec![field_a.clone()]);
-
- // create data
- let c = Int32Array::from(vec![1, 2, 3, 4, 5, 6]);
- let b_data = ArrayDataBuilder::new(field_b.data_type().clone())
- .len(6)
- .add_child_data(c.data().clone())
- .build();
- let b = StructArray::from(b_data);
- let a_data = ArrayDataBuilder::new(field_a.data_type().clone())
- .len(6)
- .add_child_data(b.data().clone())
- .build();
- let a = StructArray::from(a_data);
-
- assert_eq!(a.null_count(), 0);
- assert_eq!(a.column(0).null_count(), 0);
-
- // build a racord batch
- let batch = RecordBatch::try_new(Arc::new(schema), vec![Arc::new(a)]).unwrap();
-
- roundtrip(
- "test_arrow_writer_2_level_struct_non_null.parquet",
- batch,
- Some(SMALL_SIZE / 2),
- );
- }
-
- #[test]
- fn arrow_writer_2_level_struct_mixed_null() {
- // tests writing <struct<struct<primitive>>
- let field_c = Field::new("c", DataType::Int32, false);
- let field_b = Field::new("b", DataType::Struct(vec![field_c]), true);
- let field_a = Field::new("a", DataType::Struct(vec![field_b.clone()]), false);
- let schema = Schema::new(vec![field_a.clone()]);
-
- // create data
- let c = Int32Array::from(vec![1, 2, 3, 4, 5, 6]);
- let b_data = ArrayDataBuilder::new(field_b.data_type().clone())
- .len(6)
- .null_bit_buffer(Buffer::from(vec![0b00100111]))
- .add_child_data(c.data().clone())
- .build();
- let b = StructArray::from(b_data);
- // a intentionally has no null buffer, to test that this is handled correctly
- let a_data = ArrayDataBuilder::new(field_a.data_type().clone())
- .len(6)
- .add_child_data(b.data().clone())
- .build();
- let a = StructArray::from(a_data);
-
- assert_eq!(a.null_count(), 0);
- assert_eq!(a.column(0).null_count(), 2);
-
- // build a racord batch
- let batch = RecordBatch::try_new(Arc::new(schema), vec![Arc::new(a)]).unwrap();
-
- roundtrip(
- "test_arrow_writer_2_level_struct_mixed_null.parquet",
- batch,
- Some(SMALL_SIZE / 2),
- );
- }
-
- const SMALL_SIZE: usize = 7;
-
- fn roundtrip(
- filename: &str,
- expected_batch: RecordBatch,
- max_row_group_size: Option<usize>,
- ) -> File {
- let file = get_temp_file(filename, &[]);
-
- let mut writer = ArrowWriter::try_new(
- file.try_clone().unwrap(),
- expected_batch.schema(),
- max_row_group_size.map(|size| {
- WriterProperties::builder()
- .set_max_row_group_size(size)
- .build()
- }),
- )
- .expect("Unable to write file");
- writer.write(&expected_batch).unwrap();
- writer.close().unwrap();
-
- let reader = SerializedFileReader::new(file.try_clone().unwrap()).unwrap();
- let mut arrow_reader = ParquetFileArrowReader::new(Arc::new(reader));
- let mut record_batch_reader = arrow_reader.get_record_reader(1024).unwrap();
-
- let actual_batch = record_batch_reader
- .next()
- .expect("No batch found")
- .expect("Unable to get batch");
-
- assert_eq!(expected_batch.schema(), actual_batch.schema());
- assert_eq!(expected_batch.num_columns(), actual_batch.num_columns());
- assert_eq!(expected_batch.num_rows(), actual_batch.num_rows());
- for i in 0..expected_batch.num_columns() {
- let expected_data = expected_batch.column(i).data();
- let actual_data = actual_batch.column(i).data();
-
- assert_eq!(expected_data, actual_data);
- }
-
- file
- }
-
- fn one_column_roundtrip(
- filename: &str,
- values: ArrayRef,
- nullable: bool,
- max_row_group_size: Option<usize>,
- ) -> File {
- let schema = Schema::new(vec![Field::new(
- "col",
- values.data_type().clone(),
- nullable,
- )]);
- let expected_batch =
- RecordBatch::try_new(Arc::new(schema), vec![values]).unwrap();
-
- roundtrip(filename, expected_batch, max_row_group_size)
- }
-
- fn values_required<A, I>(iter: I, filename: &str)
- where
- A: From<Vec<I::Item>> + Array + 'static,
- I: IntoIterator,
- {
- let raw_values: Vec<_> = iter.into_iter().collect();
- let values = Arc::new(A::from(raw_values));
- one_column_roundtrip(filename, values, false, Some(SMALL_SIZE / 2));
- }
-
- fn values_optional<A, I>(iter: I, filename: &str)
- where
- A: From<Vec<Option<I::Item>>> + Array + 'static,
- I: IntoIterator,
- {
- let optional_raw_values: Vec<_> = iter
- .into_iter()
- .enumerate()
- .map(|(i, v)| if i % 2 == 0 { None } else { Some(v) })
- .collect();
- let optional_values = Arc::new(A::from(optional_raw_values));
- one_column_roundtrip(filename, optional_values, true, Some(SMALL_SIZE / 2));
- }
-
- fn required_and_optional<A, I>(iter: I, filename: &str)
- where
- A: From<Vec<I::Item>> + From<Vec<Option<I::Item>>> + Array + 'static,
- I: IntoIterator + Clone,
- {
- values_required::<A, I>(iter.clone(), filename);
- values_optional::<A, I>(iter, filename);
- }
-
- #[test]
- fn all_null_primitive_single_column() {
- let values = Arc::new(Int32Array::from(vec![None; SMALL_SIZE]));
- one_column_roundtrip(
- "all_null_primitive_single_column",
- values,
- true,
- Some(SMALL_SIZE / 2),
- );
- }
- #[test]
- fn null_single_column() {
- let values = Arc::new(NullArray::new(SMALL_SIZE));
- one_column_roundtrip("null_single_column", values, true, Some(SMALL_SIZE / 2));
- // null arrays are always nullable, a test with non-nullable nulls fails
- }
-
- #[test]
- fn bool_single_column() {
- required_and_optional::<BooleanArray, _>(
- [true, false].iter().cycle().copied().take(SMALL_SIZE),
- "bool_single_column",
- );
- }
-
- #[test]
- fn i8_single_column() {
- required_and_optional::<Int8Array, _>(0..SMALL_SIZE as i8, "i8_single_column");
- }
-
- #[test]
- fn i16_single_column() {
- required_and_optional::<Int16Array, _>(0..SMALL_SIZE as i16, "i16_single_column");
- }
-
- #[test]
- fn i32_single_column() {
- required_and_optional::<Int32Array, _>(0..SMALL_SIZE as i32, "i32_single_column");
- }
-
- #[test]
- fn i64_single_column() {
- required_and_optional::<Int64Array, _>(0..SMALL_SIZE as i64, "i64_single_column");
- }
-
- #[test]
- fn u8_single_column() {
- required_and_optional::<UInt8Array, _>(0..SMALL_SIZE as u8, "u8_single_column");
- }
-
- #[test]
- fn u16_single_column() {
- required_and_optional::<UInt16Array, _>(
- 0..SMALL_SIZE as u16,
- "u16_single_column",
- );
- }
-
- #[test]
- fn u32_single_column() {
- required_and_optional::<UInt32Array, _>(
- 0..SMALL_SIZE as u32,
- "u32_single_column",
- );
- }
-
- #[test]
- fn u64_single_column() {
- required_and_optional::<UInt64Array, _>(
- 0..SMALL_SIZE as u64,
- "u64_single_column",
- );
- }
-
- #[test]
- fn f32_single_column() {
- required_and_optional::<Float32Array, _>(
- (0..SMALL_SIZE).map(|i| i as f32),
- "f32_single_column",
- );
- }
-
- #[test]
- fn f64_single_column() {
- required_and_optional::<Float64Array, _>(
- (0..SMALL_SIZE).map(|i| i as f64),
- "f64_single_column",
- );
- }
-
- // The timestamp array types don't implement From<Vec<T>> because they need the timezone
- // argument, and they also doesn't support building from a Vec<Option<T>>, so call
- // one_column_roundtrip manually instead of calling required_and_optional for these tests.
-
- #[test]
- fn timestamp_second_single_column() {
- let raw_values: Vec<_> = (0..SMALL_SIZE as i64).collect();
- let values = Arc::new(TimestampSecondArray::from_vec(raw_values, None));
-
- one_column_roundtrip("timestamp_second_single_column", values, false, Some(3));
- }
-
- #[test]
- fn timestamp_millisecond_single_column() {
- let raw_values: Vec<_> = (0..SMALL_SIZE as i64).collect();
- let values = Arc::new(TimestampMillisecondArray::from_vec(raw_values, None));
-
- one_column_roundtrip(
- "timestamp_millisecond_single_column",
- values,
- false,
- Some(SMALL_SIZE / 2 + 1),
- );
- }
-
- #[test]
- fn timestamp_microsecond_single_column() {
- let raw_values: Vec<_> = (0..SMALL_SIZE as i64).collect();
- let values = Arc::new(TimestampMicrosecondArray::from_vec(raw_values, None));
-
- one_column_roundtrip(
- "timestamp_microsecond_single_column",
- values,
- false,
- Some(SMALL_SIZE / 2 + 2),
- );
- }
-
- #[test]
- fn timestamp_nanosecond_single_column() {
- let raw_values: Vec<_> = (0..SMALL_SIZE as i64).collect();
- let values = Arc::new(TimestampNanosecondArray::from_vec(raw_values, None));
-
- one_column_roundtrip(
- "timestamp_nanosecond_single_column",
- values,
- false,
- Some(SMALL_SIZE / 2),
- );
- }
-
- #[test]
- fn date32_single_column() {
- required_and_optional::<Date32Array, _>(
- 0..SMALL_SIZE as i32,
- "date32_single_column",
- );
- }
-
- #[test]
- fn date64_single_column() {
- // Date64 must be a multiple of 86400000, see ARROW-10925
- required_and_optional::<Date64Array, _>(
- (0..(SMALL_SIZE as i64 * 86400000)).step_by(86400000),
- "date64_single_column",
- );
- }
-
- #[test]
- fn time32_second_single_column() {
- required_and_optional::<Time32SecondArray, _>(
- 0..SMALL_SIZE as i32,
- "time32_second_single_column",
- );
- }
-
- #[test]
- fn time32_millisecond_single_column() {
- required_and_optional::<Time32MillisecondArray, _>(
- 0..SMALL_SIZE as i32,
- "time32_millisecond_single_column",
- );
- }
-
- #[test]
- fn time64_microsecond_single_column() {
- required_and_optional::<Time64MicrosecondArray, _>(
- 0..SMALL_SIZE as i64,
- "time64_microsecond_single_column",
- );
- }
-
- #[test]
- fn time64_nanosecond_single_column() {
- required_and_optional::<Time64NanosecondArray, _>(
- 0..SMALL_SIZE as i64,
- "time64_nanosecond_single_column",
- );
- }
-
- #[test]
- #[should_panic(expected = "Converting Duration to parquet not supported")]
- fn duration_second_single_column() {
- required_and_optional::<DurationSecondArray, _>(
- 0..SMALL_SIZE as i64,
- "duration_second_single_column",
- );
- }
-
- #[test]
- #[should_panic(expected = "Converting Duration to parquet not supported")]
- fn duration_millisecond_single_column() {
- required_and_optional::<DurationMillisecondArray, _>(
- 0..SMALL_SIZE as i64,
- "duration_millisecond_single_column",
- );
- }
-
- #[test]
- #[should_panic(expected = "Converting Duration to parquet not supported")]
- fn duration_microsecond_single_column() {
- required_and_optional::<DurationMicrosecondArray, _>(
- 0..SMALL_SIZE as i64,
- "duration_microsecond_single_column",
- );
- }
-
- #[test]
- #[should_panic(expected = "Converting Duration to parquet not supported")]
- fn duration_nanosecond_single_column() {
- required_and_optional::<DurationNanosecondArray, _>(
- 0..SMALL_SIZE as i64,
- "duration_nanosecond_single_column",
- );
- }
-
- #[test]
- fn interval_year_month_single_column() {
- required_and_optional::<IntervalYearMonthArray, _>(
- 0..SMALL_SIZE as i32,
- "interval_year_month_single_column",
- );
- }
-
- #[test]
- fn interval_day_time_single_column() {
- required_and_optional::<IntervalDayTimeArray, _>(
- 0..SMALL_SIZE as i64,
- "interval_day_time_single_column",
- );
- }
-
- #[test]
- fn binary_single_column() {
- let one_vec: Vec<u8> = (0..SMALL_SIZE as u8).collect();
- let many_vecs: Vec<_> = std::iter::repeat(one_vec).take(SMALL_SIZE).collect();
- let many_vecs_iter = many_vecs.iter().map(|v| v.as_slice());
-
- // BinaryArrays can't be built from Vec<Option<&str>>, so only call `values_required`
- values_required::<BinaryArray, _>(many_vecs_iter, "binary_single_column");
- }
-
- #[test]
- fn large_binary_single_column() {
- let one_vec: Vec<u8> = (0..SMALL_SIZE as u8).collect();
- let many_vecs: Vec<_> = std::iter::repeat(one_vec).take(SMALL_SIZE).collect();
- let many_vecs_iter = many_vecs.iter().map(|v| v.as_slice());
-
- // LargeBinaryArrays can't be built from Vec<Option<&str>>, so only call `values_required`
- values_required::<LargeBinaryArray, _>(
- many_vecs_iter,
- "large_binary_single_column",
- );
- }
-
- #[test]
- fn fixed_size_binary_single_column() {
- let mut builder = FixedSizeBinaryBuilder::new(16, 4);
- builder.append_value(b"0123").unwrap();
- builder.append_null().unwrap();
- builder.append_value(b"8910").unwrap();
- builder.append_value(b"1112").unwrap();
- let array = Arc::new(builder.finish());
-
- one_column_roundtrip(
- "fixed_size_binary_single_column",
- array,
- true,
- Some(SMALL_SIZE / 2),
- );
- }
-
- #[test]
- fn string_single_column() {
- let raw_values: Vec<_> = (0..SMALL_SIZE).map(|i| i.to_string()).collect();
- let raw_strs = raw_values.iter().map(|s| s.as_str());
-
- required_and_optional::<StringArray, _>(raw_strs, "string_single_column");
- }
-
- #[test]
- fn large_string_single_column() {
- let raw_values: Vec<_> = (0..SMALL_SIZE).map(|i| i.to_string()).collect();
- let raw_strs = raw_values.iter().map(|s| s.as_str());
-
- required_and_optional::<LargeStringArray, _>(
- raw_strs,
- "large_string_single_column",
- );
- }
-
- #[test]
- fn list_single_column() {
- let a_values = Int32Array::from(vec![1, 2, 3, 4, 5, 6, 7, 8, 9, 10]);
- let a_value_offsets =
- arrow::buffer::Buffer::from(&[0, 1, 3, 3, 6, 10].to_byte_slice());
- let a_list_data = ArrayData::builder(DataType::List(Box::new(Field::new(
- "item",
- DataType::Int32,
- false,
- ))))
- .len(5)
- .add_buffer(a_value_offsets)
- .null_bit_buffer(Buffer::from(vec![0b00011011]))
- .add_child_data(a_values.data().clone())
- .build();
-
- assert_eq!(a_list_data.null_count(), 1);
-
- let a = ListArray::from(a_list_data);
- let values = Arc::new(a);
-
- one_column_roundtrip("list_single_column", values, true, Some(SMALL_SIZE / 2));
- }
-
- #[test]
- fn large_list_single_column() {
- let a_values = Int32Array::from(vec![1, 2, 3, 4, 5, 6, 7, 8, 9, 10]);
- let a_value_offsets =
- arrow::buffer::Buffer::from(&[0i64, 1, 3, 3, 6, 10].to_byte_slice());
- let a_list_data = ArrayData::builder(DataType::LargeList(Box::new(Field::new(
- "large_item",
- DataType::Int32,
- true,
- ))))
- .len(5)
- .add_buffer(a_value_offsets)
- .add_child_data(a_values.data().clone())
- .null_bit_buffer(Buffer::from(vec![0b00011011]))
- .build();
-
- // I think this setup is incorrect because this should pass
- assert_eq!(a_list_data.null_count(), 1);
-
- let a = LargeListArray::from(a_list_data);
- let values = Arc::new(a);
-
- one_column_roundtrip(
- "large_list_single_column",
- values,
- true,
- Some(SMALL_SIZE / 2),
- );
- }
-
- #[test]
- fn struct_single_column() {
- let a_values = Int32Array::from(vec![1, 2, 3, 4, 5, 6, 7, 8, 9, 10]);
- let struct_field_a = Field::new("f", DataType::Int32, false);
- let s = StructArray::from(vec![(struct_field_a, Arc::new(a_values) as ArrayRef)]);
-
- let values = Arc::new(s);
- one_column_roundtrip("struct_single_column", values, false, Some(SMALL_SIZE / 2));
- }
-
- #[test]
- fn arrow_writer_string_dictionary() {
- // define schema
- let schema = Arc::new(Schema::new(vec![Field::new_dict(
- "dictionary",
- DataType::Dictionary(Box::new(DataType::Int32), Box::new(DataType::Utf8)),
- true,
- 42,
- true,
- )]));
-
- // create some data
- let d: Int32DictionaryArray = [Some("alpha"), None, Some("beta"), Some("alpha")]
- .iter()
- .copied()
- .collect();
-
- // build a record batch
- let expected_batch = RecordBatch::try_new(schema, vec![Arc::new(d)]).unwrap();
-
- roundtrip(
- "test_arrow_writer_string_dictionary.parquet",
- expected_batch,
- Some(SMALL_SIZE / 2),
- );
- }
-
- #[test]
- fn arrow_writer_primitive_dictionary() {
- // define schema
- let schema = Arc::new(Schema::new(vec![Field::new_dict(
- "dictionary",
- DataType::Dictionary(Box::new(DataType::UInt8), Box::new(DataType::UInt32)),
- true,
- 42,
- true,
- )]));
-
- // create some data
- let key_builder = PrimitiveBuilder::<UInt8Type>::new(3);
- let value_builder = PrimitiveBuilder::<UInt32Type>::new(2);
- let mut builder = PrimitiveDictionaryBuilder::new(key_builder, value_builder);
- builder.append(12345678).unwrap();
- builder.append_null().unwrap();
- builder.append(22345678).unwrap();
- builder.append(12345678).unwrap();
- let d = builder.finish();
-
- // build a record batch
- let expected_batch = RecordBatch::try_new(schema, vec![Arc::new(d)]).unwrap();
-
- roundtrip(
- "test_arrow_writer_primitive_dictionary.parquet",
- expected_batch,
- Some(SMALL_SIZE / 2),
- );
- }
-
- #[test]
- fn arrow_writer_string_dictionary_unsigned_index() {
- // define schema
- let schema = Arc::new(Schema::new(vec![Field::new_dict(
- "dictionary",
- DataType::Dictionary(Box::new(DataType::UInt8), Box::new(DataType::Utf8)),
- true,
- 42,
- true,
- )]));
-
- // create some data
- let d: UInt8DictionaryArray = [Some("alpha"), None, Some("beta"), Some("alpha")]
- .iter()
- .copied()
- .collect();
-
- // build a record batch
- let expected_batch = RecordBatch::try_new(schema, vec![Arc::new(d)]).unwrap();
-
- roundtrip(
- "test_arrow_writer_string_dictionary_unsigned_index.parquet",
- expected_batch,
- Some(SMALL_SIZE / 2),
- );
- }
-
- #[test]
- fn u32_min_max() {
- // check values roundtrip through parquet
- let values = Arc::new(UInt32Array::from_iter_values(vec![
- u32::MIN,
- u32::MIN + 1,
- (i32::MAX as u32) - 1,
- i32::MAX as u32,
- (i32::MAX as u32) + 1,
- u32::MAX - 1,
- u32::MAX,
- ]));
- let file = one_column_roundtrip("u32_min_max_single_column", values, false, None);
-
- // check statistics are valid
- let reader = SerializedFileReader::new(file).unwrap();
- let metadata = reader.metadata();
- assert_eq!(metadata.num_row_groups(), 1);
- let row_group = metadata.row_group(0);
- assert_eq!(row_group.num_columns(), 1);
- let column = row_group.column(0);
- let stats = column.statistics().unwrap();
- assert!(stats.has_min_max_set());
- if let Statistics::Int32(stats) = stats {
- assert_eq!(*stats.min() as u32, u32::MIN);
- assert_eq!(*stats.max() as u32, u32::MAX);
- } else {
- panic!("Statistics::Int32 missing")
- }
- }
-
- #[test]
- fn u64_min_max() {
- // check values roundtrip through parquet
- let values = Arc::new(UInt64Array::from_iter_values(vec![
- u64::MIN,
- u64::MIN + 1,
- (i64::MAX as u64) - 1,
- i64::MAX as u64,
- (i64::MAX as u64) + 1,
- u64::MAX - 1,
- u64::MAX,
- ]));
- let file = one_column_roundtrip("u64_min_max_single_column", values, false, None);
-
- // check statistics are valid
- let reader = SerializedFileReader::new(file).unwrap();
- let metadata = reader.metadata();
- assert_eq!(metadata.num_row_groups(), 1);
- let row_group = metadata.row_group(0);
- assert_eq!(row_group.num_columns(), 1);
- let column = row_group.column(0);
- let stats = column.statistics().unwrap();
- assert!(stats.has_min_max_set());
- if let Statistics::Int64(stats) = stats {
- assert_eq!(*stats.min() as u64, u64::MIN);
- assert_eq!(*stats.max() as u64, u64::MAX);
- } else {
- panic!("Statistics::Int64 missing")
- }
- }
-
- #[test]
- fn statistics_null_counts_only_nulls() {
- // check that null-count statistics for "only NULL"-columns are correct
- let values = Arc::new(UInt64Array::from(vec![None, None]));
- let file = one_column_roundtrip("null_counts", values, true, None);
-
- // check statistics are valid
- let reader = SerializedFileReader::new(file).unwrap();
- let metadata = reader.metadata();
- assert_eq!(metadata.num_row_groups(), 1);
- let row_group = metadata.row_group(0);
- assert_eq!(row_group.num_columns(), 1);
- let column = row_group.column(0);
- let stats = column.statistics().unwrap();
- assert_eq!(stats.null_count(), 2);
- }
-}
diff --git a/parquet/src/arrow/converter.rs b/parquet/src/arrow/converter.rs
deleted file mode 100644
index 1672be9..0000000
--- a/parquet/src/arrow/converter.rs
+++ /dev/null
@@ -1,454 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-use crate::data_type::{ByteArray, DataType, FixedLenByteArray, Int96};
-// TODO: clean up imports (best done when there are few moving parts)
-use arrow::array::{
- Array, ArrayRef, BinaryBuilder, DecimalBuilder, FixedSizeBinaryBuilder,
- IntervalDayTimeArray, IntervalDayTimeBuilder, IntervalYearMonthArray,
- IntervalYearMonthBuilder, LargeBinaryBuilder, LargeStringBuilder, PrimitiveBuilder,
- PrimitiveDictionaryBuilder, StringBuilder, StringDictionaryBuilder,
-};
-use arrow::compute::cast;
-use std::convert::{From, TryInto};
-use std::sync::Arc;
-
-use crate::errors::Result;
-use arrow::datatypes::{ArrowDictionaryKeyType, ArrowPrimitiveType};
-
-use arrow::array::{
- BinaryArray, DecimalArray, DictionaryArray, FixedSizeBinaryArray, LargeBinaryArray,
- LargeStringArray, PrimitiveArray, StringArray, TimestampNanosecondArray,
-};
-use std::marker::PhantomData;
-
-use crate::data_type::Int32Type as ParquetInt32Type;
-use arrow::datatypes::Int32Type;
-
-/// A converter is used to consume record reader's content and convert it to arrow
-/// primitive array.
-pub trait Converter<S, T> {
- /// This method converts record reader's buffered content into arrow array.
- /// It will consume record reader's data, but will not reset record reader's
- /// state.
- fn convert(&self, source: S) -> Result<T>;
-}
-
-pub struct FixedSizeArrayConverter {
- byte_width: i32,
-}
-
-impl FixedSizeArrayConverter {
- pub fn new(byte_width: i32) -> Self {
- Self { byte_width }
- }
-}
-
-impl Converter<Vec<Option<FixedLenByteArray>>, FixedSizeBinaryArray>
- for FixedSizeArrayConverter
-{
- fn convert(
- &self,
- source: Vec<Option<FixedLenByteArray>>,
- ) -> Result<FixedSizeBinaryArray> {
- let mut builder = FixedSizeBinaryBuilder::new(source.len(), self.byte_width);
- for v in source {
- match v {
- Some(array) => builder.append_value(array.data()),
- None => builder.append_null(),
- }?
- }
-
- Ok(builder.finish())
- }
-}
-
-pub struct DecimalArrayConverter {
- precision: i32,
- scale: i32,
-}
-
-impl DecimalArrayConverter {
- pub fn new(precision: i32, scale: i32) -> Self {
- Self { precision, scale }
- }
-
- fn from_bytes_to_i128(b: &[u8]) -> i128 {
- assert!(b.len() <= 16, "DecimalArray supports only up to size 16");
- let first_bit = b[0] & 128u8 == 128u8;
- let mut result = if first_bit { [255u8; 16] } else { [0u8; 16] };
- for (i, v) in b.iter().enumerate() {
- result[i + (16 - b.len())] = *v;
- }
- i128::from_be_bytes(result)
- }
-}
-
-impl Converter<Vec<Option<FixedLenByteArray>>, DecimalArray> for DecimalArrayConverter {
- fn convert(&self, source: Vec<Option<FixedLenByteArray>>) -> Result<DecimalArray> {
- let mut builder = DecimalBuilder::new(
- source.len(),
- self.precision as usize,
- self.scale as usize,
- );
- for v in source {
- match v {
- Some(array) => {
- builder.append_value(Self::from_bytes_to_i128(array.data()))
- }
- None => builder.append_null(),
- }?
- }
-
- Ok(builder.finish())
- }
-}
-/// An Arrow Interval converter, which reads the first 4 bytes of a Parquet interval,
-/// and interprets it as an i32 value representing the Arrow YearMonth value
-pub struct IntervalYearMonthArrayConverter {}
-
-impl Converter<Vec<Option<FixedLenByteArray>>, IntervalYearMonthArray>
- for IntervalYearMonthArrayConverter
-{
- fn convert(
- &self,
- source: Vec<Option<FixedLenByteArray>>,
- ) -> Result<IntervalYearMonthArray> {
- let mut builder = IntervalYearMonthBuilder::new(source.len());
- for v in source {
- match v {
- Some(array) => builder.append_value(i32::from_le_bytes(
- array.data()[0..4].try_into().unwrap(),
- )),
- None => builder.append_null(),
- }?
- }
-
- Ok(builder.finish())
- }
-}
-
-/// An Arrow Interval converter, which reads the last 8 bytes of a Parquet interval,
-/// and interprets it as an i32 value representing the Arrow DayTime value
-pub struct IntervalDayTimeArrayConverter {}
-
-impl Converter<Vec<Option<FixedLenByteArray>>, IntervalDayTimeArray>
- for IntervalDayTimeArrayConverter
-{
- fn convert(
- &self,
- source: Vec<Option<FixedLenByteArray>>,
- ) -> Result<IntervalDayTimeArray> {
- let mut builder = IntervalDayTimeBuilder::new(source.len());
- for v in source {
- match v {
- Some(array) => builder.append_value(i64::from_le_bytes(
- array.data()[4..12].try_into().unwrap(),
- )),
- None => builder.append_null(),
- }?
- }
-
- Ok(builder.finish())
- }
-}
-
-pub struct Int96ArrayConverter {
- pub timezone: Option<String>,
-}
-
-impl Converter<Vec<Option<Int96>>, TimestampNanosecondArray> for Int96ArrayConverter {
- fn convert(&self, source: Vec<Option<Int96>>) -> Result<TimestampNanosecondArray> {
- Ok(TimestampNanosecondArray::from_opt_vec(
- source
- .into_iter()
- .map(|int96| int96.map(|val| val.to_i64() * 1_000_000))
- .collect(),
- self.timezone.clone(),
- ))
- }
-}
-
-pub struct Utf8ArrayConverter {}
-
-impl Converter<Vec<Option<ByteArray>>, StringArray> for Utf8ArrayConverter {
- fn convert(&self, source: Vec<Option<ByteArray>>) -> Result<StringArray> {
- let data_size = source
- .iter()
- .map(|x| x.as_ref().map(|b| b.len()).unwrap_or(0))
- .sum();
-
- let mut builder = StringBuilder::with_capacity(source.len(), data_size);
- for v in source {
- match v {
- Some(array) => builder.append_value(array.as_utf8()?),
- None => builder.append_null(),
- }?
- }
-
- Ok(builder.finish())
- }
-}
-
-pub struct LargeUtf8ArrayConverter {}
-
-impl Converter<Vec<Option<ByteArray>>, LargeStringArray> for LargeUtf8ArrayConverter {
- fn convert(&self, source: Vec<Option<ByteArray>>) -> Result<LargeStringArray> {
- let data_size = source
- .iter()
- .map(|x| x.as_ref().map(|b| b.len()).unwrap_or(0))
- .sum();
-
- let mut builder = LargeStringBuilder::with_capacity(source.len(), data_size);
- for v in source {
- match v {
- Some(array) => builder.append_value(array.as_utf8()?),
- None => builder.append_null(),
- }?
- }
-
- Ok(builder.finish())
- }
-}
-
-pub struct BinaryArrayConverter {}
-
-impl Converter<Vec<Option<ByteArray>>, BinaryArray> for BinaryArrayConverter {
- fn convert(&self, source: Vec<Option<ByteArray>>) -> Result<BinaryArray> {
- let mut builder = BinaryBuilder::new(source.len());
- for v in source {
- match v {
- Some(array) => builder.append_value(array.data()),
- None => builder.append_null(),
- }?
- }
-
- Ok(builder.finish())
- }
-}
-
-pub struct LargeBinaryArrayConverter {}
-
-impl Converter<Vec<Option<ByteArray>>, LargeBinaryArray> for LargeBinaryArrayConverter {
- fn convert(&self, source: Vec<Option<ByteArray>>) -> Result<LargeBinaryArray> {
- let mut builder = LargeBinaryBuilder::new(source.len());
- for v in source {
- match v {
- Some(array) => builder.append_value(array.data()),
- None => builder.append_null(),
- }?
- }
-
- Ok(builder.finish())
- }
-}
-
-pub struct StringDictionaryArrayConverter {}
-
-impl<K: ArrowDictionaryKeyType> Converter<Vec<Option<ByteArray>>, DictionaryArray<K>>
- for StringDictionaryArrayConverter
-{
- fn convert(&self, source: Vec<Option<ByteArray>>) -> Result<DictionaryArray<K>> {
- let data_size = source
- .iter()
- .map(|x| x.as_ref().map(|b| b.len()).unwrap_or(0))
- .sum();
-
- let keys_builder = PrimitiveBuilder::<K>::new(source.len());
- let values_builder = StringBuilder::with_capacity(source.len(), data_size);
-
- let mut builder = StringDictionaryBuilder::new(keys_builder, values_builder);
- for v in source {
- match v {
- Some(array) => {
- let _ = builder.append(array.as_utf8()?)?;
- }
- None => builder.append_null()?,
- }
- }
-
- Ok(builder.finish())
- }
-}
-
-pub struct DictionaryArrayConverter<DictValueSourceType, DictValueTargetType, ParquetType>
-{
- _dict_value_source_marker: PhantomData<DictValueSourceType>,
- _dict_value_target_marker: PhantomData<DictValueTargetType>,
- _parquet_marker: PhantomData<ParquetType>,
-}
-
-impl<DictValueSourceType, DictValueTargetType, ParquetType>
- DictionaryArrayConverter<DictValueSourceType, DictValueTargetType, ParquetType>
-{
- pub fn new() -> Self {
- Self {
- _dict_value_source_marker: PhantomData,
- _dict_value_target_marker: PhantomData,
- _parquet_marker: PhantomData,
- }
- }
-}
-
-impl<K, DictValueSourceType, DictValueTargetType, ParquetType>
- Converter<Vec<Option<<ParquetType as DataType>::T>>, DictionaryArray<K>>
- for DictionaryArrayConverter<DictValueSourceType, DictValueTargetType, ParquetType>
-where
- K: ArrowPrimitiveType,
- DictValueSourceType: ArrowPrimitiveType,
- DictValueTargetType: ArrowPrimitiveType,
- ParquetType: DataType,
- PrimitiveArray<DictValueSourceType>: From<Vec<Option<<ParquetType as DataType>::T>>>,
-{
- fn convert(
- &self,
- source: Vec<Option<<ParquetType as DataType>::T>>,
- ) -> Result<DictionaryArray<K>> {
- let keys_builder = PrimitiveBuilder::<K>::new(source.len());
- let values_builder = PrimitiveBuilder::<DictValueTargetType>::new(source.len());
-
- let mut builder = PrimitiveDictionaryBuilder::new(keys_builder, values_builder);
-
- let source_array: Arc<dyn Array> =
- Arc::new(PrimitiveArray::<DictValueSourceType>::from(source));
- let target_array = cast(&source_array, &DictValueTargetType::DATA_TYPE)?;
- let target = target_array
- .as_any()
- .downcast_ref::<PrimitiveArray<DictValueTargetType>>()
- .unwrap();
-
- for i in 0..target.len() {
- if target.is_null(i) {
- builder.append_null()?;
- } else {
- let _ = builder.append(target.value(i))?;
- }
- }
-
- Ok(builder.finish())
- }
-}
-
-pub type Utf8Converter =
- ArrayRefConverter<Vec<Option<ByteArray>>, StringArray, Utf8ArrayConverter>;
-pub type LargeUtf8Converter =
- ArrayRefConverter<Vec<Option<ByteArray>>, LargeStringArray, LargeUtf8ArrayConverter>;
-pub type BinaryConverter =
- ArrayRefConverter<Vec<Option<ByteArray>>, BinaryArray, BinaryArrayConverter>;
-pub type LargeBinaryConverter = ArrayRefConverter<
- Vec<Option<ByteArray>>,
- LargeBinaryArray,
- LargeBinaryArrayConverter,
->;
-pub type StringDictionaryConverter<T> = ArrayRefConverter<
- Vec<Option<ByteArray>>,
- DictionaryArray<T>,
- StringDictionaryArrayConverter,
->;
-pub type DictionaryConverter<K, SV, TV, P> = ArrayRefConverter<
- Vec<Option<<P as DataType>::T>>,
- DictionaryArray<K>,
- DictionaryArrayConverter<SV, TV, P>,
->;
-pub type PrimitiveDictionaryConverter<K, V> = ArrayRefConverter<
- Vec<Option<<ParquetInt32Type as DataType>::T>>,
- DictionaryArray<K>,
- DictionaryArrayConverter<Int32Type, V, ParquetInt32Type>,
->;
-
-pub type Int96Converter =
- ArrayRefConverter<Vec<Option<Int96>>, TimestampNanosecondArray, Int96ArrayConverter>;
-
-pub type FixedLenBinaryConverter = ArrayRefConverter<
- Vec<Option<FixedLenByteArray>>,
- FixedSizeBinaryArray,
- FixedSizeArrayConverter,
->;
-pub type IntervalYearMonthConverter = ArrayRefConverter<
- Vec<Option<FixedLenByteArray>>,
- IntervalYearMonthArray,
- IntervalYearMonthArrayConverter,
->;
-pub type IntervalDayTimeConverter = ArrayRefConverter<
- Vec<Option<FixedLenByteArray>>,
- IntervalDayTimeArray,
- IntervalDayTimeArrayConverter,
->;
-
-pub type DecimalConverter = ArrayRefConverter<
- Vec<Option<FixedLenByteArray>>,
- DecimalArray,
- DecimalArrayConverter,
->;
-
-pub struct FromConverter<S, T> {
- _source: PhantomData<S>,
- _dest: PhantomData<T>,
-}
-
-impl<S, T> FromConverter<S, T>
-where
- T: From<S>,
-{
- pub fn new() -> Self {
- Self {
- _source: PhantomData,
- _dest: PhantomData,
- }
- }
-}
-
-impl<S, T> Converter<S, T> for FromConverter<S, T>
-where
- T: From<S>,
-{
- fn convert(&self, source: S) -> Result<T> {
- Ok(T::from(source))
- }
-}
-
-pub struct ArrayRefConverter<S, A, C> {
- _source: PhantomData<S>,
- _array: PhantomData<A>,
- converter: C,
-}
-
-impl<S, A, C> ArrayRefConverter<S, A, C>
-where
- A: Array + 'static,
- C: Converter<S, A> + 'static,
-{
- pub fn new(converter: C) -> Self {
- Self {
- _source: PhantomData,
- _array: PhantomData,
- converter,
- }
- }
-}
-
-impl<S, A, C> Converter<S, ArrayRef> for ArrayRefConverter<S, A, C>
-where
- A: Array + 'static,
- C: Converter<S, A> + 'static,
-{
- fn convert(&self, source: S) -> Result<ArrayRef> {
- self.converter
- .convert(source)
- .map(|array| Arc::new(array) as ArrayRef)
- }
-}
diff --git a/parquet/src/arrow/levels.rs b/parquet/src/arrow/levels.rs
deleted file mode 100644
index 2e95039..0000000
--- a/parquet/src/arrow/levels.rs
+++ /dev/null
@@ -1,1548 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-//! Parquet definition and repetition levels
-//!
-//! Contains the algorithm for computing definition and repetition levels.
-//! The algorithm works by tracking the slots of an array that should
-//! ultimately be populated when writing to Parquet.
-//! Parquet achieves nesting through definition levels and repetition levels \[1\].
-//! Definition levels specify how many optional fields in the part for the column
-//! are defined.
-//! Repetition levels specify at what repeated field (list) in the path a column
-//! is defined.
-//!
-//! In a nested data structure such as `a.b.c`, one can see levels as defining
-//! whether a record is defined at `a`, `a.b`, or `a.b.c`.
-//! Optional fields are nullable fields, thus if all 3 fields
-//! are nullable, the maximum definition could be = 3 if there are no lists.
-//!
-//! The algorithm in this module computes the necessary information to enable
-//! the writer to keep track of which columns are at which levels, and to extract
-//! the correct values at the correct slots from Arrow arrays.
-//!
-//! It works by walking a record batch's arrays, keeping track of what values
-//! are non-null, their positions and computing what their levels are.
-//!
-//! \[1\] [parquet-format#nested-encoding](https://github.com/apache/parquet-format#nested-encoding)
-
-use arrow::array::{make_array, ArrayRef, StructArray};
-use arrow::datatypes::{DataType, Field};
-
-/// Keeps track of the level information per array that is needed to write an Arrow array to Parquet.
-///
-/// When a nested schema is traversed, intermediate [LevelInfo] structs are created to track
-/// the state of parent arrays. When a primitive Arrow array is encountered, a final [LevelInfo]
-/// is created, and this is what is used to index into the array when writing data to Parquet.
-#[derive(Debug, Eq, PartialEq, Clone)]
-pub(crate) struct LevelInfo {
- /// Array's definition levels
- pub definition: Vec<i16>,
- /// Array's optional repetition levels
- pub repetition: Option<Vec<i16>>,
- /// Array's offsets, 64-bit is used to accommodate large offset arrays
- pub array_offsets: Vec<i64>,
- // TODO: Convert to an Arrow Buffer after ARROW-10766 is merged.
- /// Array's logical validity mask, whcih gets unpacked for list children.
- /// If the parent of an array is null, all children are logically treated as
- /// null. This mask keeps track of that.
- ///
- pub array_mask: Vec<bool>,
- /// The maximum definition at this level, 0 at the record batch
- pub max_definition: i16,
- /// The type of array represented by this level info
- pub level_type: LevelType,
- /// The offset of the current level's array
- pub offset: usize,
- /// The length of the current level's array
- pub length: usize,
-}
-
-/// LevelType defines the type of level, and whether it is nullable or not
-#[derive(Debug, Eq, PartialEq, Clone, Copy)]
-pub(crate) enum LevelType {
- Root,
- List(bool),
- Struct(bool),
- Primitive(bool),
-}
-
-impl LevelType {
- #[inline]
- const fn level_increment(&self) -> i16 {
- match self {
- LevelType::Root => 0,
- LevelType::List(is_nullable)
- | LevelType::Struct(is_nullable)
- | LevelType::Primitive(is_nullable) => *is_nullable as i16,
- }
- }
-}
-
-impl LevelInfo {
- /// Create a new [LevelInfo] by filling `length` slots, and setting an initial offset.
- ///
- /// This is a convenience function to populate the starting point of the traversal.
- pub(crate) fn new(offset: usize, length: usize) -> Self {
- Self {
- // a batch has no definition level yet
- definition: vec![0; length],
- // a batch has no repetition as it is not a list
- repetition: None,
- // a batch has sequential offsets, should be num_rows + 1
- array_offsets: (0..=(length as i64)).collect(),
- // all values at a batch-level are non-null
- array_mask: vec![true; length],
- max_definition: 0,
- level_type: LevelType::Root,
- offset,
- length,
- }
- }
-
- /// Compute nested levels of the Arrow array, recursing into lists and structs.
- ///
- /// Returns a list of `LevelInfo`, where each level is for nested primitive arrays.
- ///
- /// The parent struct's nullness is tracked, as it determines whether the child
- /// max_definition should be incremented.
- /// The 'is_parent_struct' variable asks "is this field's parent a struct?".
- /// * If we are starting at a [RecordBatch], this is `false`.
- /// * If we are calculating a list's child, this is `false`.
- /// * If we are calculating a struct (i.e. `field.data_type90 == Struct`),
- /// this depends on whether the struct is a child of a struct.
- /// * If we are calculating a field inside a [StructArray], this is 'true'.
- pub(crate) fn calculate_array_levels(
- &self,
- array: &ArrayRef,
- field: &Field,
- ) -> Vec<Self> {
- let (array_offsets, array_mask) =
- Self::get_array_offsets_and_masks(array, self.offset, self.length);
- match array.data_type() {
- DataType::Null => vec![Self {
- definition: self.definition.clone(),
- repetition: self.repetition.clone(),
- array_offsets,
- array_mask,
- max_definition: self.max_definition.max(1),
- // Null type is always nullable
- level_type: LevelType::Primitive(true),
- offset: self.offset,
- length: self.length,
- }],
- DataType::Boolean
- | DataType::Int8
- | DataType::Int16
- | DataType::Int32
- | DataType::Int64
- | DataType::UInt8
- | DataType::UInt16
- | DataType::UInt32
- | DataType::UInt64
- | DataType::Float16
- | DataType::Float32
- | DataType::Float64
- | DataType::Utf8
- | DataType::LargeUtf8
- | DataType::Timestamp(_, _)
- | DataType::Date32
- | DataType::Date64
- | DataType::Time32(_)
- | DataType::Time64(_)
- | DataType::Duration(_)
- | DataType::Interval(_)
- | DataType::Binary
- | DataType::LargeBinary
- | DataType::Decimal(_, _)
- | DataType::FixedSizeBinary(_) => {
- // we return a vector of 1 value to represent the primitive
- vec![self.calculate_child_levels(
- array_offsets,
- array_mask,
- LevelType::Primitive(field.is_nullable()),
- )]
- }
- DataType::List(list_field) | DataType::LargeList(list_field) => {
- let child_offset = array_offsets[0] as usize;
- let child_len = *array_offsets.last().unwrap() as usize;
- // Calculate the list level
- let list_level = self.calculate_child_levels(
- array_offsets,
- array_mask,
- LevelType::List(field.is_nullable()),
- );
-
- // Construct the child array of the list, and get its offset + mask
- let array_data = array.data();
- let child_data = array_data.child_data().get(0).unwrap();
- let child_array = make_array(child_data.clone());
- let (child_offsets, child_mask) = Self::get_array_offsets_and_masks(
- &child_array,
- child_offset,
- child_len - child_offset,
- );
-
- match child_array.data_type() {
- // TODO: The behaviour of a <list<null>> is untested
- DataType::Null => vec![list_level],
- DataType::Boolean
- | DataType::Int8
- | DataType::Int16
- | DataType::Int32
- | DataType::Int64
- | DataType::UInt8
- | DataType::UInt16
- | DataType::UInt32
- | DataType::UInt64
- | DataType::Float16
- | DataType::Float32
- | DataType::Float64
- | DataType::Timestamp(_, _)
- | DataType::Date32
- | DataType::Date64
- | DataType::Time32(_)
- | DataType::Time64(_)
- | DataType::Duration(_)
- | DataType::Interval(_)
- | DataType::Binary
- | DataType::LargeBinary
- | DataType::Utf8
- | DataType::LargeUtf8
- | DataType::Dictionary(_, _)
- | DataType::Decimal(_, _)
- | DataType::FixedSizeBinary(_) => {
- vec![list_level.calculate_child_levels(
- child_offsets,
- child_mask,
- LevelType::Primitive(list_field.is_nullable()),
- )]
- }
- DataType::List(_) | DataType::LargeList(_) | DataType::Struct(_) => {
- list_level.calculate_array_levels(&child_array, list_field)
- }
- DataType::FixedSizeList(_, _) => unimplemented!(),
- DataType::Union(_) => unimplemented!(),
- }
- }
- DataType::FixedSizeList(_, _) => unimplemented!(),
- DataType::Struct(struct_fields) => {
- let struct_array: &StructArray = array
- .as_any()
- .downcast_ref::<StructArray>()
- .expect("Unable to get struct array");
- let struct_level = self.calculate_child_levels(
- array_offsets,
- array_mask,
- LevelType::Struct(field.is_nullable()),
- );
- let mut struct_levels = vec![];
- struct_array
- .columns()
- .into_iter()
- .zip(struct_fields)
- .for_each(|(child_array, child_field)| {
- let mut levels =
- struct_level.calculate_array_levels(child_array, child_field);
- struct_levels.append(&mut levels);
- });
- struct_levels
- }
- DataType::Union(_) => unimplemented!(),
- DataType::Dictionary(_, _) => {
- // Need to check for these cases not implemented in C++:
- // - "Writing DictionaryArray with nested dictionary type not yet supported"
- // - "Writing DictionaryArray with null encoded in dictionary type not yet supported"
- // vec![self.get_primitive_def_levels(array, field, array_mask)]
- vec![self.calculate_child_levels(
- array_offsets,
- array_mask,
- LevelType::Primitive(field.is_nullable()),
- )]
- }
- }
- }
-
- /// Calculate child/leaf array levels.
- ///
- /// The algorithm works by incrementing definitions of array values based on whether:
- /// - a value is optional or required (is_nullable)
- /// - a list value is repeated + optional or required (is_list)
- ///
- /// A record batch always starts at a populated definition = level 0.
- /// When a batch only has a primitive, i.e. `<batch<primitive[a]>>, column `a`
- /// can only have a maximum level of 1 if it is not null.
- /// If it is not null, we increment by 1, such that the null slots will = level 1.
- /// The above applies to types that have no repetition (anything not a list or map).
- ///
- /// If a batch has lists, then we increment by up to 2 levels:
- /// - 1 level for the list (repeated)
- /// - 1 level if the list itself is nullable (optional)
- ///
- /// A list's child then gets incremented using the above rules.
- ///
- /// *Exceptions*
- ///
- /// There are 2 exceptions from the above rules:
- ///
- /// 1. When at the root of the schema: We always increment the
- /// level regardless of whether the child is nullable or not. If we do not do
- /// this, we could have a non-nullable array having a definition of 0.
- ///
- /// 2. List parent, non-list child: We always increment the level in this case,
- /// regardless of whether the child is nullable or not.
- ///
- /// *Examples*
- ///
- /// A batch with only a primitive that's non-nullable. `<primitive[required]>`:
- /// * We don't increment the definition level as the array is not optional.
- /// * This would leave us with a definition of 0, so the first exception applies.
- /// * The definition level becomes 1.
- ///
- /// A batch with only a primitive that's nullable. `<primitive[optional]>`:
- /// * The definition level becomes 1, as we increment it once.
- ///
- /// A batch with a single non-nullable list (both list and child not null):
- /// * We calculate the level twice, for the list, and for the child.
- /// * At the list, the level becomes 1, where 0 indicates that the list is
- /// empty, and 1 says it's not (determined through offsets).
- /// * At the primitive level, the second exception applies. The level becomes 2.
- fn calculate_child_levels(
- &self,
- // we use 64-bit offsets to also accommodate large arrays
- array_offsets: Vec<i64>,
- array_mask: Vec<bool>,
- level_type: LevelType,
- ) -> Self {
- let min_len = *(array_offsets.last().unwrap()) as usize;
- let mut definition = Vec::with_capacity(min_len);
- let mut repetition = Vec::with_capacity(min_len);
- let mut merged_array_mask = Vec::with_capacity(min_len);
-
- let max_definition = match (self.level_type, level_type) {
- (LevelType::Root, LevelType::Struct(is_nullable)) => {
- // If the struct is non-nullable, its def level doesn't increment
- is_nullable as i16
- }
- (LevelType::Root, _) => 1,
- (_, LevelType::Root) => {
- unreachable!("Cannot have a root as a child")
- }
- (LevelType::List(_), _) => {
- self.max_definition + 1 + level_type.level_increment()
- }
- (LevelType::Struct(_), _) => {
- self.max_definition + level_type.level_increment()
- }
- (_, LevelType::List(is_nullable)) => {
- // if the child is a list, even if its parent is a root
- self.max_definition + 1 + is_nullable as i16
- }
- (LevelType::Primitive(_), _) => {
- unreachable!("Cannot have a primitive parent for any type")
- }
- };
-
- match (self.level_type, level_type) {
- (LevelType::List(_), LevelType::List(is_nullable)) => {
- // parent is a list or descendant of a list, and child is a list
- let reps = self.repetition.clone().unwrap();
- // Calculate the 2 list hierarchy definitions in advance
- // List is not empty, but null
- let l2 = max_definition - is_nullable as i16;
- // List is not empty, and not null
- let l3 = max_definition;
-
- let mut nulls_seen = 0;
-
- self.array_offsets.windows(2).for_each(|w| {
- let start = w[0] as usize;
- let end = w[1] as usize;
- let parent_len = end - start;
-
- if parent_len == 0 {
- // If the parent length is 0, there won't be a slot for the child
- let index = start + nulls_seen - self.offset;
- definition.push(self.definition[index]);
- repetition.push(0);
- merged_array_mask.push(self.array_mask[index]);
- nulls_seen += 1;
- } else {
- (start..end).for_each(|parent_index| {
- let index = parent_index + nulls_seen - self.offset;
- let parent_index = parent_index - self.offset;
-
- // parent is either defined at this level, or earlier
- let parent_def = self.definition[index];
- let parent_rep = reps[index];
- let parent_mask = self.array_mask[index];
-
- // valid parent, index into children
- let child_start = array_offsets[parent_index] as usize;
- let child_end = array_offsets[parent_index + 1] as usize;
- let child_len = child_end - child_start;
- let child_mask = array_mask[parent_index];
- let merged_mask = parent_mask && child_mask;
-
- if child_len == 0 {
- definition.push(parent_def);
- repetition.push(parent_rep);
- merged_array_mask.push(merged_mask);
- } else {
- (child_start..child_end).for_each(|child_index| {
- let rep = match (
- parent_index == start,
- child_index == child_start,
- ) {
- (true, true) => parent_rep,
- (true, false) => parent_rep + 2,
- (false, true) => parent_rep,
- (false, false) => parent_rep + 1,
- };
-
- definition.push(if !parent_mask {
- parent_def
- } else if child_mask {
- l3
- } else {
- l2
- });
- repetition.push(rep);
- merged_array_mask.push(merged_mask);
- });
- }
- });
- }
- });
-
- debug_assert_eq!(definition.len(), merged_array_mask.len());
-
- let offset = *array_offsets.first().unwrap() as usize;
- let length = *array_offsets.last().unwrap() as usize - offset;
-
- Self {
- definition,
- repetition: Some(repetition),
- array_offsets,
- array_mask: merged_array_mask,
- max_definition,
- level_type,
- offset: offset + self.offset,
- length,
- }
- }
- (LevelType::List(_), _) => {
- // List and primitive (or struct).
- // The list can have more values than the primitive, indicating that there
- // are slots where the list is empty. We use a counter to track this behaviour.
- let mut nulls_seen = 0;
-
- // let child_max_definition = list_max_definition + is_nullable as i16;
- // child values are a function of parent list offsets
- let reps = self.repetition.as_deref().unwrap();
- self.array_offsets.windows(2).for_each(|w| {
- let start = w[0] as usize;
- let end = w[1] as usize;
- let parent_len = end - start;
-
- if parent_len == 0 {
- let index = start + nulls_seen - self.offset;
- definition.push(self.definition[index]);
- repetition.push(reps[index]);
- merged_array_mask.push(self.array_mask[index]);
- nulls_seen += 1;
- } else {
- // iterate through the array, adjusting child definitions for nulls
- (start..end).for_each(|child_index| {
- let index = child_index + nulls_seen - self.offset;
- let child_mask = array_mask[child_index - self.offset];
- let parent_mask = self.array_mask[index];
- let parent_def = self.definition[index];
-
- if !parent_mask || parent_def < self.max_definition {
- definition.push(parent_def);
- repetition.push(reps[index]);
- merged_array_mask.push(parent_mask);
- } else {
- definition.push(max_definition - !child_mask as i16);
- repetition.push(reps[index]);
- merged_array_mask.push(child_mask);
- }
- });
- }
- });
-
- debug_assert_eq!(definition.len(), merged_array_mask.len());
-
- let offset = *array_offsets.first().unwrap() as usize;
- let length = *array_offsets.last().unwrap() as usize - offset;
-
- Self {
- definition,
- repetition: Some(repetition),
- array_offsets: self.array_offsets.clone(),
- array_mask: merged_array_mask,
- max_definition,
- level_type,
- offset: offset + self.offset,
- length,
- }
- }
- (_, LevelType::List(is_nullable)) => {
- // Encountering a list for the first time.
- // Calculate the 2 list hierarchy definitions in advance
-
- // List is not empty, but null (if nullable)
- let l2 = max_definition - is_nullable as i16;
- // List is not empty, and not null
- let l3 = max_definition;
-
- self.definition
- .iter()
- .enumerate()
- .for_each(|(parent_index, def)| {
- let child_from = array_offsets[parent_index];
- let child_to = array_offsets[parent_index + 1];
- let child_len = child_to - child_from;
- let child_mask = array_mask[parent_index];
- let parent_mask = self.array_mask[parent_index];
-
- match (parent_mask, child_len) {
- (true, 0) => {
- // empty slot that is valid, i.e. {"parent": {"child": [] } }
- definition.push(if child_mask { l3 } else { l2 });
- repetition.push(0);
- merged_array_mask.push(child_mask);
- }
- (false, 0) => {
- definition.push(*def);
- repetition.push(0);
- merged_array_mask.push(child_mask);
- }
- (true, _) => {
- (child_from..child_to).for_each(|child_index| {
- definition.push(if child_mask { l3 } else { l2 });
- // mark the first child slot as 0, and the next as 1
- repetition.push(if child_index == child_from {
- 0
- } else {
- 1
- });
- merged_array_mask.push(child_mask);
- });
- }
- (false, _) => {
- (child_from..child_to).for_each(|child_index| {
- definition.push(*def);
- // mark the first child slot as 0, and the next as 1
- repetition.push(if child_index == child_from {
- 0
- } else {
- 1
- });
- merged_array_mask.push(false);
- });
- }
- }
- });
-
- debug_assert_eq!(definition.len(), merged_array_mask.len());
-
- let offset = *array_offsets.first().unwrap() as usize;
- let length = *array_offsets.last().unwrap() as usize - offset;
-
- Self {
- definition,
- repetition: Some(repetition),
- array_offsets,
- array_mask: merged_array_mask,
- max_definition,
- level_type,
- offset,
- length,
- }
- }
- (_, _) => {
- self.definition
- .iter()
- .zip(array_mask.into_iter().zip(&self.array_mask))
- .for_each(|(current_def, (child_mask, parent_mask))| {
- merged_array_mask.push(*parent_mask && child_mask);
- match (parent_mask, child_mask) {
- (true, true) => {
- definition.push(max_definition);
- }
- (true, false) => {
- // The child is only legally null if its array is nullable.
- // Thus parent's max_definition is lower
- definition.push(if *current_def <= self.max_definition {
- *current_def
- } else {
- self.max_definition
- });
- }
- // if the parent was false, retain its definitions
- (false, _) => {
- definition.push(*current_def);
- }
- }
- });
-
- debug_assert_eq!(definition.len(), merged_array_mask.len());
-
- Self {
- definition,
- repetition: self.repetition.clone(), // it's None
- array_offsets,
- array_mask: merged_array_mask,
- max_definition,
- level_type,
- // Inherit parent offset and length
- offset: self.offset,
- length: self.length,
- }
- }
- }
- }
-
- /// Get the offsets of an array as 64-bit values, and validity masks as booleans
- /// - Primitive, binary and struct arrays' offsets will be a sequence, masks obtained
- /// from validity bitmap
- /// - List array offsets will be the value offsets, masks are computed from offsets
- fn get_array_offsets_and_masks(
- array: &ArrayRef,
- offset: usize,
- len: usize,
- ) -> (Vec<i64>, Vec<bool>) {
- match array.data_type() {
- DataType::Null
- | DataType::Boolean
- | DataType::Int8
- | DataType::Int16
- | DataType::Int32
- | DataType::Int64
- | DataType::UInt8
- | DataType::UInt16
- | DataType::UInt32
- | DataType::UInt64
- | DataType::Float16
- | DataType::Float32
- | DataType::Float64
- | DataType::Timestamp(_, _)
- | DataType::Date32
- | DataType::Date64
- | DataType::Time32(_)
- | DataType::Time64(_)
- | DataType::Duration(_)
- | DataType::Interval(_)
- | DataType::Binary
- | DataType::LargeBinary
- | DataType::Utf8
- | DataType::LargeUtf8
- | DataType::Struct(_)
- | DataType::Dictionary(_, _)
- | DataType::Decimal(_, _) => {
- let array_mask = match array.data().null_buffer() {
- Some(buf) => get_bool_array_slice(buf, array.offset() + offset, len),
- None => vec![true; len],
- };
- ((0..=(len as i64)).collect(), array_mask)
- }
- DataType::List(_) => {
- let data = array.data();
- let offsets = unsafe { data.buffers()[0].typed_data::<i32>() };
- let offsets = offsets
- .to_vec()
- .into_iter()
- .skip(offset)
- .take(len + 1)
- .map(|v| v as i64)
- .collect::<Vec<i64>>();
- let array_mask = match array.data().null_buffer() {
- Some(buf) => get_bool_array_slice(buf, array.offset() + offset, len),
- None => vec![true; len],
- };
- (offsets, array_mask)
- }
- DataType::LargeList(_) => {
- let offsets = unsafe { array.data().buffers()[0].typed_data::<i64>() }
- .iter()
- .skip(offset)
- .take(len + 1)
- .copied()
- .collect();
- let array_mask = match array.data().null_buffer() {
- Some(buf) => get_bool_array_slice(buf, array.offset() + offset, len),
- None => vec![true; len],
- };
- (offsets, array_mask)
- }
- DataType::FixedSizeBinary(value_len) => {
- let array_mask = match array.data().null_buffer() {
- Some(buf) => get_bool_array_slice(buf, array.offset() + offset, len),
- None => vec![true; len],
- };
- let value_len = *value_len as i64;
- (
- (0..=(len as i64)).map(|v| v * value_len).collect(),
- array_mask,
- )
- }
- DataType::FixedSizeList(_, _) | DataType::Union(_) => {
- unimplemented!("Getting offsets not yet implemented")
- }
- }
- }
-
- /// Given a level's information, calculate the offsets required to index an array correctly.
- pub(crate) fn filter_array_indices(&self) -> Vec<usize> {
- // happy path if not dealing with lists
- let is_nullable = match self.level_type {
- LevelType::Primitive(is_nullable) => is_nullable,
- _ => panic!(
- "Cannot filter indices on a non-primitive array, found {:?}",
- self.level_type
- ),
- };
- if self.repetition.is_none() {
- return self
- .definition
- .iter()
- .enumerate()
- .filter_map(|(i, def)| {
- if *def == self.max_definition {
- Some(i)
- } else {
- None
- }
- })
- .collect();
- }
- let mut filtered = vec![];
- // remove slots that are false from definition_mask
- let mut index = 0;
- self.definition.iter().for_each(|def| {
- if *def == self.max_definition {
- filtered.push(index);
- }
- if *def >= self.max_definition - is_nullable as i16 {
- index += 1;
- }
- });
- filtered
- }
-}
-
-/// Convert an Arrow buffer to a boolean array slice
-/// TODO: this was created for buffers, so might not work for bool array, might be slow too
-#[inline]
-fn get_bool_array_slice(
- buffer: &arrow::buffer::Buffer,
- offset: usize,
- len: usize,
-) -> Vec<bool> {
- let data = buffer.as_slice();
- (offset..(len + offset))
- .map(|i| arrow::util::bit_util::get_bit(data, i))
- .collect()
-}
-
-#[cfg(test)]
-mod tests {
- use super::*;
-
- use std::sync::Arc;
-
- use arrow::array::*;
- use arrow::buffer::Buffer;
- use arrow::datatypes::{Schema, ToByteSlice};
- use arrow::record_batch::RecordBatch;
-
- #[test]
- fn test_calculate_array_levels_twitter_example() {
- // based on the example at https://blog.twitter.com/engineering/en_us/a/2013/dremel-made-simple-with-parquet.html
- // [[a, b, c], [d, e, f, g]], [[h], [i,j]]
- let parent_levels = LevelInfo {
- definition: vec![0, 0],
- repetition: None,
- array_offsets: vec![0, 1, 2], // 2 records, root offsets always sequential
- array_mask: vec![true, true], // both lists defined
- max_definition: 0,
- level_type: LevelType::Root,
- offset: 0,
- length: 2,
- };
- // offset into array, each level1 has 2 values
- let array_offsets = vec![0, 2, 4];
- let array_mask = vec![true, true];
-
- // calculate level1 levels
- let levels = parent_levels.calculate_child_levels(
- array_offsets.clone(),
- array_mask,
- LevelType::List(false),
- );
- //
- let expected_levels = LevelInfo {
- definition: vec![1, 1, 1, 1],
- repetition: Some(vec![0, 1, 0, 1]),
- array_offsets,
- array_mask: vec![true, true, true, true],
- max_definition: 1,
- level_type: LevelType::List(false),
- offset: 0,
- length: 4,
- };
- // the separate asserts make it easier to see what's failing
- assert_eq!(&levels.definition, &expected_levels.definition);
- assert_eq!(&levels.repetition, &expected_levels.repetition);
- assert_eq!(&levels.array_mask, &expected_levels.array_mask);
- assert_eq!(&levels.array_offsets, &expected_levels.array_offsets);
- assert_eq!(&levels.max_definition, &expected_levels.max_definition);
- assert_eq!(&levels.level_type, &expected_levels.level_type);
- // this assert is to help if there are more variables added to the struct
- assert_eq!(&levels, &expected_levels);
-
- // level2
- let parent_levels = levels;
- let array_offsets = vec![0, 3, 7, 8, 10];
- let array_mask = vec![true, true, true, true];
- let levels = parent_levels.calculate_child_levels(
- array_offsets.clone(),
- array_mask,
- LevelType::List(false),
- );
- let expected_levels = LevelInfo {
- definition: vec![2, 2, 2, 2, 2, 2, 2, 2, 2, 2],
- repetition: Some(vec![0, 2, 2, 1, 2, 2, 2, 0, 1, 2]),
- array_offsets,
- array_mask: vec![true; 10],
- max_definition: 2,
- level_type: LevelType::List(false),
- offset: 0,
- length: 10,
- };
- assert_eq!(&levels.definition, &expected_levels.definition);
- assert_eq!(&levels.repetition, &expected_levels.repetition);
- assert_eq!(&levels.array_mask, &expected_levels.array_mask);
- assert_eq!(&levels.max_definition, &expected_levels.max_definition);
- assert_eq!(&levels.array_offsets, &expected_levels.array_offsets);
- assert_eq!(&levels.level_type, &expected_levels.level_type);
- assert_eq!(&levels, &expected_levels);
- }
-
- #[test]
- fn test_calculate_one_level_1() {
- // This test calculates the levels for a non-null primitive array
- let parent_levels = LevelInfo {
- definition: vec![0; 10],
- repetition: None,
- array_offsets: (0..=10).collect(),
- array_mask: vec![true; 10],
- max_definition: 0,
- level_type: LevelType::Root,
- offset: 0,
- length: 10,
- };
- let array_offsets: Vec<i64> = (0..=10).collect();
- let array_mask = vec![true; 10];
-
- let levels = parent_levels.calculate_child_levels(
- array_offsets.clone(),
- array_mask.clone(),
- LevelType::Primitive(false),
- );
- let expected_levels = LevelInfo {
- definition: vec![1; 10],
- repetition: None,
- array_offsets,
- array_mask,
- max_definition: 1,
- level_type: LevelType::Primitive(false),
- offset: 0,
- length: 10,
- };
- assert_eq!(&levels, &expected_levels);
- }
-
- #[test]
- fn test_calculate_one_level_2() {
- // This test calculates the levels for a non-null primitive array
- let parent_levels = LevelInfo {
- definition: vec![0; 5],
- repetition: None,
- array_offsets: (0..=5).collect(),
- array_mask: vec![true, true, true, true, true],
- max_definition: 0,
- level_type: LevelType::Root,
- offset: 0,
- length: 5,
- };
- let array_offsets: Vec<i64> = (0..=5).collect();
- let array_mask = vec![true, false, true, true, false];
-
- let levels = parent_levels.calculate_child_levels(
- array_offsets.clone(),
- array_mask.clone(),
- LevelType::Primitive(true),
- );
- let expected_levels = LevelInfo {
- definition: vec![1, 0, 1, 1, 0],
- repetition: None,
- array_offsets,
- array_mask,
- max_definition: 1,
- level_type: LevelType::Primitive(true),
- offset: 0,
- length: 5,
- };
- assert_eq!(&levels, &expected_levels);
- }
-
- #[test]
- fn test_calculate_array_levels_1() {
- // if all array values are defined (e.g. batch<list<_>>)
- // [[0], [1], [2], [3], [4]]
- let parent_levels = LevelInfo {
- definition: vec![0; 5],
- repetition: None,
- array_offsets: vec![0, 1, 2, 3, 4, 5],
- array_mask: vec![true, true, true, true, true],
- max_definition: 0,
- level_type: LevelType::Root,
- offset: 0,
- length: 5,
- };
- let array_offsets = vec![0, 2, 2, 4, 8, 11];
- let array_mask = vec![true, false, true, true, true];
-
- let levels = parent_levels.calculate_child_levels(
- array_offsets.clone(),
- array_mask,
- LevelType::List(true),
- );
- // array: [[0, 0], _1_, [2, 2], [3, 3, 3, 3], [4, 4, 4]]
- // all values are defined as we do not have nulls on the root (batch)
- // repetition:
- // 0: 0, 1
- // 1:
- // 2: 0, 1
- // 3: 0, 1, 1, 1
- // 4: 0, 1, 1
- let expected_levels = LevelInfo {
- // The levels are normally 2 because we:
- // - Calculate the level at the list
- // - Calculate the level at the list's child
- // We do not do this in these tests, thus the levels are 1 less.
- definition: vec![1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1],
- repetition: Some(vec![0, 1, 0, 0, 1, 0, 1, 1, 1, 0, 1, 1]),
- array_offsets,
- array_mask: vec![
- true, true, false, true, true, true, true, true, true, true, true, true,
- ],
- max_definition: 1,
- level_type: LevelType::List(true),
- offset: 0,
- length: 11, // the child has 11 slots
- };
- assert_eq!(&levels.definition, &expected_levels.definition);
- assert_eq!(&levels.repetition, &expected_levels.repetition);
- assert_eq!(&levels.array_offsets, &expected_levels.array_offsets);
- assert_eq!(&levels.max_definition, &expected_levels.max_definition);
- assert_eq!(&levels.level_type, &expected_levels.level_type);
- assert_eq!(&levels, &expected_levels);
- }
-
- #[test]
- fn test_calculate_array_levels_2() {
- // If some values are null
- //
- // This emulates an array in the form: <struct<list<?>>
- // with values:
- // - 0: [0, 1], but is null because of the struct
- // - 1: []
- // - 2: [2, 3], but is null because of the struct
- // - 3: [4, 5, 6, 7]
- // - 4: [8, 9, 10]
- //
- // If the first values of a list are null due to a parent, we have to still account for them
- // while indexing, because they would affect the way the child is indexed
- // i.e. in the above example, we have to know that [0, 1] has to be skipped
- let parent_levels = LevelInfo {
- definition: vec![0, 1, 0, 1, 1],
- repetition: None,
- array_offsets: vec![0, 1, 2, 3, 4, 5],
- array_mask: vec![false, true, false, true, true],
- max_definition: 1,
- level_type: LevelType::Struct(true),
- offset: 0,
- length: 5,
- };
- let array_offsets = vec![0, 2, 2, 4, 8, 11];
- let array_mask = vec![true, false, true, true, true];
-
- let levels = parent_levels.calculate_child_levels(
- array_offsets.clone(),
- array_mask,
- LevelType::List(true),
- );
- let expected_levels = LevelInfo {
- // 0 1 [2] are 0 (not defined at level 1)
- // [2] is 1, but has 0 slots so is not populated (defined at level 1 only)
- // 2 3 [4] are 0
- // 4 5 6 7 [8] are 1 (defined at level 1 only)
- // 8 9 10 [11] are 2 (defined at both levels)
- definition: vec![0, 0, 1, 0, 0, 2, 2, 2, 2, 2, 2, 2],
- repetition: Some(vec![0, 1, 0, 0, 1, 0, 1, 1, 1, 0, 1, 1]),
- array_offsets,
- array_mask: vec![
- false, false, false, false, false, true, true, true, true, true, true,
- true,
- ],
- max_definition: 2,
- level_type: LevelType::List(true),
- offset: 0,
- length: 11,
- };
- assert_eq!(&levels.definition, &expected_levels.definition);
- assert_eq!(&levels.repetition, &expected_levels.repetition);
- assert_eq!(&levels.array_offsets, &expected_levels.array_offsets);
- assert_eq!(&levels.max_definition, &expected_levels.max_definition);
- assert_eq!(&levels.level_type, &expected_levels.level_type);
- assert_eq!(&levels, &expected_levels);
-
- // nested lists (using previous test)
- let nested_parent_levels = levels;
- let array_offsets = vec![0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22];
- let array_mask = vec![
- true, true, true, true, true, true, true, true, true, true, true,
- ];
- let levels = nested_parent_levels.calculate_child_levels(
- array_offsets.clone(),
- array_mask,
- LevelType::List(true),
- );
- let expected_levels = LevelInfo {
- // (def: 0) 0 1 [2] are 0 (take parent)
- // (def: 0) 2 3 [4] are 0 (take parent)
- // (def: 0) 4 5 [6] are 0 (take parent)
- // (def: 0) 6 7 [8] are 0 (take parent)
- // (def: 1) 8 9 [10] are 1 (take parent)
- // (def: 1) 10 11 [12] are 1 (take parent)
- // (def: 1) 12 23 [14] are 1 (take parent)
- // (def: 1) 14 15 [16] are 1 (take parent)
- // (def: 2) 16 17 [18] are 2 (defined at all levels)
- // (def: 2) 18 19 [20] are 2 (defined at all levels)
- // (def: 2) 20 21 [22] are 2 (defined at all levels)
- //
- // 0 1 [2] are 0 (not defined at level 1)
- // [2] is 1, but has 0 slots so is not populated (defined at level 1 only)
- // 2 3 [4] are 0
- // 4 5 6 7 [8] are 1 (defined at level 1 only)
- // 8 9 10 [11] are 2 (defined at both levels)
- //
- // 0: [[100, 101], [102, 103]]
- // 1: []
- // 2: [[104, 105], [106, 107]]
- // 3: [[108, 109], [110, 111], [112, 113], [114, 115]]
- // 4: [[116, 117], [118, 119], [120, 121]]
- definition: vec![
- 0, 0, 0, 0, 1, 0, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4,
- ],
- repetition: Some(vec![
- 0, 2, 1, 2, 0, 0, 2, 1, 2, 0, 2, 1, 2, 1, 2, 1, 2, 0, 2, 1, 2, 1, 2,
- ]),
- array_offsets,
- array_mask: vec![
- false, false, false, false, false, false, false, false, false, true,
- true, true, true, true, true, true, true, true, true, true, true, true,
- true,
- ],
- max_definition: 4,
- level_type: LevelType::List(true),
- offset: 0,
- length: 22,
- };
- assert_eq!(&levels.definition, &expected_levels.definition);
- assert_eq!(&levels.repetition, &expected_levels.repetition);
- assert_eq!(&levels.array_offsets, &expected_levels.array_offsets);
- assert_eq!(&levels.array_mask, &expected_levels.array_mask);
- assert_eq!(&levels.max_definition, &expected_levels.max_definition);
- assert_eq!(&levels.level_type, &expected_levels.level_type);
- assert_eq!(&levels, &expected_levels);
- }
-
- #[test]
- fn test_calculate_array_levels_nested_list() {
- // if all array values are defined (e.g. batch<list<_>>)
- // The array at this level looks like:
- // 0: [a]
- // 1: [a]
- // 2: [a]
- // 3: [a]
- let parent_levels = LevelInfo {
- definition: vec![1, 1, 1, 1],
- repetition: None,
- array_offsets: vec![0, 1, 2, 3, 4],
- array_mask: vec![true, true, true, true],
- max_definition: 1,
- level_type: LevelType::Struct(true),
- offset: 0,
- length: 4,
- };
- // 0: null ([], but mask is false, so it's not just an empty list)
- // 1: [1, 2, 3]
- // 2: [4, 5]
- // 3: [6, 7]
- let array_offsets = vec![0, 1, 4, 6, 8];
- let array_mask = vec![false, true, true, true];
-
- let levels = parent_levels.calculate_child_levels(
- array_offsets.clone(),
- array_mask,
- LevelType::List(true),
- );
- // 0: [null], level 1 is defined, but not 2
- // 1: [1, 2, 3]
- // 2: [4, 5]
- // 3: [6, 7]
- let expected_levels = LevelInfo {
- definition: vec![1, 2, 2, 2, 2, 2, 2, 2],
- repetition: Some(vec![0, 0, 1, 1, 0, 1, 0, 1]),
- array_offsets,
- array_mask: vec![false, true, true, true, true, true, true, true],
- max_definition: 2,
- level_type: LevelType::List(true),
- offset: 0,
- length: 8,
- };
- assert_eq!(&levels.definition, &expected_levels.definition);
- assert_eq!(&levels.repetition, &expected_levels.repetition);
- assert_eq!(&levels.array_offsets, &expected_levels.array_offsets);
- assert_eq!(&levels.max_definition, &expected_levels.max_definition);
- assert_eq!(&levels.level_type, &expected_levels.level_type);
- assert_eq!(&levels, &expected_levels);
-
- // nested lists (using previous test)
- let nested_parent_levels = levels;
- // 0: [null] (was a populated null slot at the parent)
- // 1: [201]
- // 2: [202, 203]
- // 3: null ([])
- // 4: [204, 205, 206]
- // 5: [207, 208, 209, 210]
- // 6: [] (tests a non-null empty list slot)
- // 7: [211, 212, 213, 214, 215]
- let array_offsets = vec![0, 1, 2, 4, 4, 7, 11, 11, 16];
- // logically, the fist slot of the mask is false
- let array_mask = vec![true, true, true, false, true, true, true, true];
- let levels = nested_parent_levels.calculate_child_levels(
- array_offsets.clone(),
- array_mask,
- LevelType::List(true),
- );
- // We have 7 array values, and at least 15 primitives (from array_offsets)
- // 0: (-)[null], parent was null, no value populated here
- // 1: (0)[201], (1)[202, 203], (2)[[null]]
- // 2: (3)[204, 205, 206], (4)[207, 208, 209, 210]
- // 3: (5)[[]], (6)[211, 212, 213, 214, 215]
- //
- // In a JSON syntax with the schema: <struct<list<list<primitive>>>>, this translates into:
- // 0: {"struct": [ null ]}
- // 1: {"struct": [ [201], [202, 203], [] ]}
- // 2: {"struct": [ [204, 205, 206], [207, 208, 209, 210] ]}
- // 3: {"struct": [ [], [211, 212, 213, 214, 215] ]}
- let expected_levels = LevelInfo {
- definition: vec![1, 4, 4, 4, 2, 4, 4, 4, 4, 4, 4, 4, 2, 4, 4, 4, 4, 4],
- repetition: Some(vec![0, 0, 1, 2, 1, 0, 2, 2, 1, 2, 2, 2, 0, 1, 2, 2, 2, 2]),
- array_mask: vec![
- false, true, true, true, false, true, true, true, true, true, true, true,
- true, true, true, true, true, true,
- ],
- array_offsets,
- max_definition: 4,
- level_type: LevelType::List(true),
- offset: 0,
- length: 16,
- };
- assert_eq!(&levels.definition, &expected_levels.definition);
- assert_eq!(&levels.repetition, &expected_levels.repetition);
- assert_eq!(&levels.array_offsets, &expected_levels.array_offsets);
- assert_eq!(&levels.array_mask, &expected_levels.array_mask);
- assert_eq!(&levels.max_definition, &expected_levels.max_definition);
- assert_eq!(&levels.level_type, &expected_levels.level_type);
- assert_eq!(&levels, &expected_levels);
- }
-
- #[test]
- fn test_calculate_nested_struct_levels() {
- // tests a <struct[a]<struct[b]<int[c]>>
- // array:
- // - {a: {b: {c: 1}}}
- // - {a: {b: {c: null}}}
- // - {a: {b: {c: 3}}}
- // - {a: {b: null}}
- // - {a: null}}
- // - {a: {b: {c: 6}}}
- let a_levels = LevelInfo {
- definition: vec![1, 1, 1, 1, 0, 1],
- repetition: None,
- array_offsets: (0..=6).collect(),
- array_mask: vec![true, true, true, true, false, true],
- max_definition: 1,
- level_type: LevelType::Struct(true),
- offset: 0,
- length: 6,
- };
- // b's offset and mask
- let b_offsets: Vec<i64> = (0..=6).collect();
- let b_mask = vec![true, true, true, false, false, true];
- // b's expected levels
- let b_expected_levels = LevelInfo {
- definition: vec![2, 2, 2, 1, 0, 2],
- repetition: None,
- array_offsets: (0..=6).collect(),
- array_mask: vec![true, true, true, false, false, true],
- max_definition: 2,
- level_type: LevelType::Struct(true),
- offset: 0,
- length: 6,
- };
- let b_levels = a_levels.calculate_child_levels(
- b_offsets.clone(),
- b_mask,
- LevelType::Struct(true),
- );
- assert_eq!(&b_expected_levels, &b_levels);
-
- // c's offset and mask
- let c_offsets = b_offsets;
- let c_mask = vec![true, false, true, false, false, true];
- // c's expected levels
- let c_expected_levels = LevelInfo {
- definition: vec![3, 2, 3, 1, 0, 3],
- repetition: None,
- array_offsets: c_offsets.clone(),
- array_mask: vec![true, false, true, false, false, true],
- max_definition: 3,
- level_type: LevelType::Struct(true),
- offset: 0,
- length: 6,
- };
- let c_levels =
- b_levels.calculate_child_levels(c_offsets, c_mask, LevelType::Struct(true));
- assert_eq!(&c_expected_levels, &c_levels);
- }
-
- #[test]
- fn list_single_column() {
- // this tests the level generation from the arrow_writer equivalent test
-
- let a_values = Int32Array::from(vec![1, 2, 3, 4, 5, 6, 7, 8, 9, 10]);
- let a_value_offsets =
- arrow::buffer::Buffer::from(&[0, 1, 3, 3, 6, 10].to_byte_slice());
- let a_list_type =
- DataType::List(Box::new(Field::new("item", DataType::Int32, true)));
- let a_list_data = ArrayData::builder(a_list_type.clone())
- .len(5)
- .add_buffer(a_value_offsets)
- .null_bit_buffer(Buffer::from(vec![0b00011011]))
- .add_child_data(a_values.data().clone())
- .build();
-
- assert_eq!(a_list_data.null_count(), 1);
-
- let a = ListArray::from(a_list_data);
- let values = Arc::new(a);
-
- let schema = Schema::new(vec![Field::new("item", a_list_type, true)]);
-
- let batch = RecordBatch::try_new(Arc::new(schema), vec![values]).unwrap();
-
- let expected_batch_level = LevelInfo {
- definition: vec![0; 2],
- repetition: None,
- array_offsets: (0..=2).collect(),
- array_mask: vec![true, true],
- max_definition: 0,
- level_type: LevelType::Root,
- offset: 2,
- length: 2,
- };
-
- let batch_level = LevelInfo::new(2, 2);
- assert_eq!(&batch_level, &expected_batch_level);
-
- // calculate the list's level
- let mut levels = vec![];
- batch
- .columns()
- .iter()
- .zip(batch.schema().fields())
- .for_each(|(array, field)| {
- let mut array_levels = batch_level.calculate_array_levels(array, field);
- levels.append(&mut array_levels);
- });
- assert_eq!(levels.len(), 1);
-
- let list_level = levels.get(0).unwrap();
-
- let expected_level = LevelInfo {
- definition: vec![0, 3, 3, 3],
- repetition: Some(vec![0, 0, 1, 1]),
- array_offsets: vec![3, 3, 6],
- array_mask: vec![false, true, true, true],
- max_definition: 3,
- level_type: LevelType::Primitive(true),
- offset: 3,
- length: 3,
- };
- assert_eq!(&list_level.definition, &expected_level.definition);
- assert_eq!(&list_level.repetition, &expected_level.repetition);
- assert_eq!(&list_level.array_offsets, &expected_level.array_offsets);
- assert_eq!(&list_level.array_mask, &expected_level.array_mask);
- assert_eq!(&list_level.max_definition, &expected_level.max_definition);
- assert_eq!(&list_level.level_type, &expected_level.level_type);
- assert_eq!(list_level, &expected_level);
- }
-
- #[test]
- fn mixed_struct_list() {
- // this tests the level generation from the equivalent arrow_writer_complex test
-
- // define schema
- let struct_field_d = Field::new("d", DataType::Float64, true);
- let struct_field_f = Field::new("f", DataType::Float32, true);
- let struct_field_g = Field::new(
- "g",
- DataType::List(Box::new(Field::new("items", DataType::Int16, false))),
- false,
- );
- let struct_field_e = Field::new(
- "e",
- DataType::Struct(vec![struct_field_f.clone(), struct_field_g.clone()]),
- true,
- );
- let schema = Schema::new(vec![
- Field::new("a", DataType::Int32, false),
- Field::new("b", DataType::Int32, true),
- Field::new(
- "c",
- DataType::Struct(vec![struct_field_d.clone(), struct_field_e.clone()]),
- true, // https://github.com/apache/arrow-rs/issues/245
- ),
- ]);
-
- // create some data
- let a = Int32Array::from(vec![1, 2, 3, 4, 5]);
- let b = Int32Array::from(vec![Some(1), None, None, Some(4), Some(5)]);
- let d = Float64Array::from(vec![None, None, None, Some(1.0), None]);
- let f = Float32Array::from(vec![Some(0.0), None, Some(333.3), None, Some(5.25)]);
-
- let g_value = Int16Array::from(vec![1, 2, 3, 4, 5, 6, 7, 8, 9, 10]);
-
- // Construct a buffer for value offsets, for the nested array:
- // [[1], [2, 3], null, [4, 5, 6], [7, 8, 9, 10]]
- let g_value_offsets =
- arrow::buffer::Buffer::from(&[0, 1, 3, 3, 6, 10].to_byte_slice());
-
- // Construct a list array from the above two
- let g_list_data = ArrayData::builder(struct_field_g.data_type().clone())
- .len(5)
- .add_buffer(g_value_offsets)
- .add_child_data(g_value.data().clone())
- .build();
- let g = ListArray::from(g_list_data);
-
- let e = StructArray::from(vec![
- (struct_field_f, Arc::new(f) as ArrayRef),
- (struct_field_g, Arc::new(g) as ArrayRef),
- ]);
-
- let c = StructArray::from(vec![
- (struct_field_d, Arc::new(d) as ArrayRef),
- (struct_field_e, Arc::new(e) as ArrayRef),
- ]);
-
- // build a record batch
- let batch = RecordBatch::try_new(
- Arc::new(schema),
- vec![Arc::new(a), Arc::new(b), Arc::new(c)],
- )
- .unwrap();
-
- //////////////////////////////////////////////
- let expected_batch_level = LevelInfo {
- definition: vec![0; 5],
- repetition: None,
- array_offsets: (0..=5).collect(),
- array_mask: vec![true, true, true, true, true],
- max_definition: 0,
- level_type: LevelType::Root,
- offset: 0,
- length: 5,
- };
-
- let batch_level = LevelInfo::new(0, 5);
- assert_eq!(&batch_level, &expected_batch_level);
-
- // calculate the list's level
- let mut levels = vec![];
- batch
- .columns()
- .iter()
- .zip(batch.schema().fields())
- .for_each(|(array, field)| {
- let mut array_levels = batch_level.calculate_array_levels(array, field);
- levels.append(&mut array_levels);
- });
- assert_eq!(levels.len(), 5);
-
- // test "a" levels
- let list_level = levels.get(0).unwrap();
-
- let expected_level = LevelInfo {
- definition: vec![1, 1, 1, 1, 1],
- repetition: None,
- array_offsets: vec![0, 1, 2, 3, 4, 5],
- array_mask: vec![true, true, true, true, true],
- max_definition: 1,
- level_type: LevelType::Primitive(false),
- offset: 0,
- length: 5,
- };
- assert_eq!(list_level, &expected_level);
-
- // test "b" levels
- let list_level = levels.get(1).unwrap();
-
- let expected_level = LevelInfo {
- definition: vec![1, 0, 0, 1, 1],
- repetition: None,
- array_offsets: vec![0, 1, 2, 3, 4, 5],
- array_mask: vec![true, false, false, true, true],
- max_definition: 1,
- level_type: LevelType::Primitive(true),
- offset: 0,
- length: 5,
- };
- assert_eq!(list_level, &expected_level);
-
- // test "d" levels
- let list_level = levels.get(2).unwrap();
-
- let expected_level = LevelInfo {
- definition: vec![1, 1, 1, 2, 1],
- repetition: None,
- array_offsets: vec![0, 1, 2, 3, 4, 5],
- array_mask: vec![false, false, false, true, false],
- max_definition: 2,
- level_type: LevelType::Primitive(true),
- offset: 0,
- length: 5,
- };
- assert_eq!(list_level, &expected_level);
-
- // test "f" levels
- let list_level = levels.get(3).unwrap();
-
- let expected_level = LevelInfo {
- definition: vec![3, 2, 3, 2, 3],
- repetition: None,
- array_offsets: vec![0, 1, 2, 3, 4, 5],
- array_mask: vec![true, false, true, false, true],
- max_definition: 3,
- level_type: LevelType::Primitive(true),
- offset: 0,
- length: 5,
- };
- assert_eq!(list_level, &expected_level);
- }
-
- #[test]
- fn test_filter_array_indices() {
- let level = LevelInfo {
- definition: vec![3, 3, 3, 1, 3, 3, 3],
- repetition: Some(vec![0, 1, 1, 0, 0, 1, 1]),
- array_offsets: vec![0, 3, 3, 6],
- array_mask: vec![true, true, true, false, true, true, true],
- max_definition: 3,
- level_type: LevelType::Primitive(true),
- offset: 0,
- length: 6,
- };
-
- let expected = vec![0, 1, 2, 3, 4, 5];
- let filter = level.filter_array_indices();
- assert_eq!(expected, filter);
- }
-
- #[test]
- fn test_null_vs_nonnull_struct() {
- // define schema
- let offset_field = Field::new("offset", DataType::Int32, true);
- let schema = Schema::new(vec![Field::new(
- "some_nested_object",
- DataType::Struct(vec![offset_field.clone()]),
- false,
- )]);
-
- // create some data
- let offset = Int32Array::from(vec![1, 2, 3, 4, 5]);
-
- let some_nested_object =
- StructArray::from(vec![(offset_field, Arc::new(offset) as ArrayRef)]);
-
- // build a record batch
- let batch =
- RecordBatch::try_new(Arc::new(schema), vec![Arc::new(some_nested_object)])
- .unwrap();
-
- let batch_level = LevelInfo::new(0, batch.num_rows());
- let struct_null_level =
- batch_level.calculate_array_levels(batch.column(0), batch.schema().field(0));
-
- // create second batch
- // define schema
- let offset_field = Field::new("offset", DataType::Int32, true);
- let schema = Schema::new(vec![Field::new(
- "some_nested_object",
- DataType::Struct(vec![offset_field.clone()]),
- true,
- )]);
-
- // create some data
- let offset = Int32Array::from(vec![1, 2, 3, 4, 5]);
-
- let some_nested_object =
- StructArray::from(vec![(offset_field, Arc::new(offset) as ArrayRef)]);
-
- // build a record batch
- let batch =
- RecordBatch::try_new(Arc::new(schema), vec![Arc::new(some_nested_object)])
- .unwrap();
-
- let batch_level = LevelInfo::new(0, batch.num_rows());
- let struct_non_null_level =
- batch_level.calculate_array_levels(batch.column(0), batch.schema().field(0));
-
- // The 2 levels should not be the same
- if struct_non_null_level == struct_null_level {
- panic!("Levels should not be equal, to reflect the difference in struct nullness");
- }
- }
-}
diff --git a/parquet/src/arrow/mod.rs b/parquet/src/arrow/mod.rs
deleted file mode 100644
index afb1fdc..0000000
--- a/parquet/src/arrow/mod.rs
+++ /dev/null
@@ -1,70 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-//! [Apache Arrow](http://arrow.apache.org/) is a cross-language development platform for
-//! in-memory data.
-//!
-//! This mod provides API for converting between arrow and parquet.
-//!
-//! # Example of reading parquet file into arrow record batch
-//!
-//! ```rust, no_run
-//! use arrow::record_batch::RecordBatchReader;
-//! use parquet::file::reader::SerializedFileReader;
-//! use parquet::arrow::{ParquetFileArrowReader, ArrowReader};
-//! use std::sync::Arc;
-//! use std::fs::File;
-//!
-//! let file = File::open("parquet.file").unwrap();
-//! let file_reader = SerializedFileReader::new(file).unwrap();
-//! let mut arrow_reader = ParquetFileArrowReader::new(Arc::new(file_reader));
-//!
-//! println!("Converted arrow schema is: {}", arrow_reader.get_schema().unwrap());
-//! println!("Arrow schema after projection is: {}",
-//! arrow_reader.get_schema_by_columns(vec![2, 4, 6], true).unwrap());
-//!
-//! let mut record_batch_reader = arrow_reader.get_record_reader(2048).unwrap();
-//!
-//! for maybe_record_batch in record_batch_reader {
-//! let record_batch = maybe_record_batch.unwrap();
-//! if record_batch.num_rows() > 0 {
-//! println!("Read {} records.", record_batch.num_rows());
-//! } else {
-//! println!("End of file!");
-//! }
-//!}
-//! ```
-
-pub mod array_reader;
-pub mod arrow_array_reader;
-pub mod arrow_reader;
-pub mod arrow_writer;
-pub mod converter;
-pub(in crate::arrow) mod levels;
-pub(in crate::arrow) mod record_reader;
-pub mod schema;
-
-pub use self::arrow_reader::ArrowReader;
-pub use self::arrow_reader::ParquetFileArrowReader;
-pub use self::arrow_writer::ArrowWriter;
-pub use self::schema::{
- arrow_to_parquet_schema, parquet_to_arrow_schema, parquet_to_arrow_schema_by_columns,
- parquet_to_arrow_schema_by_root_columns,
-};
-
-/// Schema metadata key used to store serialized Arrow IPC schema
-pub const ARROW_SCHEMA_META_KEY: &str = "ARROW:schema";
diff --git a/parquet/src/arrow/record_reader.rs b/parquet/src/arrow/record_reader.rs
deleted file mode 100644
index 4dd7da9..0000000
--- a/parquet/src/arrow/record_reader.rs
+++ /dev/null
@@ -1,802 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-use std::cmp::{max, min};
-use std::mem::{replace, size_of};
-
-use crate::column::{page::PageReader, reader::ColumnReaderImpl};
-use crate::data_type::DataType;
-use crate::errors::{ParquetError, Result};
-use crate::schema::types::ColumnDescPtr;
-use arrow::array::BooleanBufferBuilder;
-use arrow::bitmap::Bitmap;
-use arrow::buffer::{Buffer, MutableBuffer};
-
-const MIN_BATCH_SIZE: usize = 1024;
-
-/// A `RecordReader` is a stateful column reader that delimits semantic records.
-pub struct RecordReader<T: DataType> {
- column_desc: ColumnDescPtr,
-
- records: MutableBuffer,
- def_levels: Option<MutableBuffer>,
- rep_levels: Option<MutableBuffer>,
- null_bitmap: Option<BooleanBufferBuilder>,
- column_reader: Option<ColumnReaderImpl<T>>,
-
- /// Number of records accumulated in records
- num_records: usize,
- /// Number of values `num_records` contains.
- num_values: usize,
-
- values_seen: usize,
- /// Starts from 1, number of values have been written to buffer
- values_written: usize,
- in_middle_of_record: bool,
-}
-
-impl<T: DataType> RecordReader<T> {
- pub fn new(column_schema: ColumnDescPtr) -> Self {
- let (def_levels, null_map) = if column_schema.max_def_level() > 0 {
- (
- Some(MutableBuffer::new(MIN_BATCH_SIZE)),
- Some(BooleanBufferBuilder::new(0)),
- )
- } else {
- (None, None)
- };
-
- let rep_levels = if column_schema.max_rep_level() > 0 {
- Some(MutableBuffer::new(MIN_BATCH_SIZE))
- } else {
- None
- };
-
- Self {
- records: MutableBuffer::new(MIN_BATCH_SIZE),
- def_levels,
- rep_levels,
- null_bitmap: null_map,
- column_reader: None,
- column_desc: column_schema,
- num_records: 0,
- num_values: 0,
- values_seen: 0,
- values_written: 0,
- in_middle_of_record: false,
- }
- }
-
- /// Set the current page reader.
- pub fn set_page_reader(&mut self, page_reader: Box<dyn PageReader>) -> Result<()> {
- self.column_reader =
- Some(ColumnReaderImpl::new(self.column_desc.clone(), page_reader));
- Ok(())
- }
-
- /// Try to read `num_records` of column data into internal buffer.
- ///
- /// # Returns
- ///
- /// Number of actual records read.
- pub fn read_records(&mut self, num_records: usize) -> Result<usize> {
- if self.column_reader.is_none() {
- return Ok(0);
- }
-
- let mut records_read = 0;
-
- // Used to mark whether we have reached the end of current
- // column chunk
- let mut end_of_column = false;
-
- loop {
- // Try to find some records from buffers that has been read into memory
- // but not counted as seen records.
- records_read += self.split_records(num_records - records_read)?;
-
- // Since page reader contains complete records, so if we reached end of a
- // page reader, we should reach the end of a record
- if end_of_column
- && self.values_seen >= self.values_written
- && self.in_middle_of_record
- {
- self.num_records += 1;
- self.num_values = self.values_seen;
- self.in_middle_of_record = false;
- records_read += 1;
- }
-
- if (records_read >= num_records) || end_of_column {
- break;
- }
-
- let batch_size = max(num_records - records_read, MIN_BATCH_SIZE);
-
- // Try to more value from parquet pages
- let values_read = self.read_one_batch(batch_size)?;
- if values_read < batch_size {
- end_of_column = true;
- }
- }
-
- Ok(records_read)
- }
-
- /// Returns number of records stored in buffer.
- pub fn num_records(&self) -> usize {
- self.num_records
- }
-
- /// Return number of values stored in buffer.
- /// If the parquet column is not repeated, it should be equals to `num_records`,
- /// otherwise it should be larger than or equal to `num_records`.
- pub fn num_values(&self) -> usize {
- self.num_values
- }
-
- /// Returns definition level data.
- /// The implementation has side effects. It will create a new buffer to hold those
- /// definition level values that have already been read into memory but not counted
- /// as record values, e.g. those from `self.num_values` to `self.values_written`.
- pub fn consume_def_levels(&mut self) -> Result<Option<Buffer>> {
- let new_buffer = if let Some(ref mut def_levels_buf) = &mut self.def_levels {
- let num_left_values = self.values_written - self.num_values;
- // create an empty buffer, as it will be resized below
- let mut new_buffer = MutableBuffer::new(0);
- let num_bytes = num_left_values * size_of::<i16>();
- let new_len = self.num_values * size_of::<i16>();
-
- new_buffer.resize(num_bytes, 0);
-
- let new_def_levels = new_buffer.as_slice_mut();
- let left_def_levels = &def_levels_buf.as_slice_mut()[new_len..];
-
- new_def_levels[0..num_bytes].copy_from_slice(&left_def_levels[0..num_bytes]);
-
- def_levels_buf.resize(new_len, 0);
- Some(new_buffer)
- } else {
- None
- };
-
- Ok(replace(&mut self.def_levels, new_buffer).map(|x| x.into()))
- }
-
- /// Return repetition level data.
- /// The side effect is similar to `consume_def_levels`.
- pub fn consume_rep_levels(&mut self) -> Result<Option<Buffer>> {
- // TODO: Optimize to reduce the copy
- let new_buffer = if let Some(ref mut rep_levels_buf) = &mut self.rep_levels {
- let num_left_values = self.values_written - self.num_values;
- // create an empty buffer, as it will be resized below
- let mut new_buffer = MutableBuffer::new(0);
- let num_bytes = num_left_values * size_of::<i16>();
- let new_len = self.num_values * size_of::<i16>();
-
- new_buffer.resize(num_bytes, 0);
-
- let new_rep_levels = new_buffer.as_slice_mut();
- let left_rep_levels = &rep_levels_buf.as_slice_mut()[new_len..];
-
- new_rep_levels[0..num_bytes].copy_from_slice(&left_rep_levels[0..num_bytes]);
-
- rep_levels_buf.resize(new_len, 0);
-
- Some(new_buffer)
- } else {
- None
- };
-
- Ok(replace(&mut self.rep_levels, new_buffer).map(|x| x.into()))
- }
-
- /// Returns currently stored buffer data.
- /// The side effect is similar to `consume_def_levels`.
- pub fn consume_record_data(&mut self) -> Result<Buffer> {
- // TODO: Optimize to reduce the copy
- let num_left_values = self.values_written - self.num_values;
- // create an empty buffer, as it will be resized below
- let mut new_buffer = MutableBuffer::new(0);
- let num_bytes = num_left_values * T::get_type_size();
- let new_len = self.num_values * T::get_type_size();
-
- new_buffer.resize(num_bytes, 0);
-
- let new_records = new_buffer.as_slice_mut();
- let left_records = &mut self.records.as_slice_mut()[new_len..];
-
- new_records[0..num_bytes].copy_from_slice(&left_records[0..num_bytes]);
-
- self.records.resize(new_len, 0);
-
- Ok(replace(&mut self.records, new_buffer).into())
- }
-
- /// Returns currently stored null bitmap data.
- /// The side effect is similar to `consume_def_levels`.
- pub fn consume_bitmap_buffer(&mut self) -> Result<Option<Buffer>> {
- // TODO: Optimize to reduce the copy
- if self.column_desc.max_def_level() > 0 {
- assert!(self.null_bitmap.is_some());
- let num_left_values = self.values_written - self.num_values;
- let new_bitmap_builder = Some(BooleanBufferBuilder::new(max(
- MIN_BATCH_SIZE,
- num_left_values,
- )));
-
- let old_bitmap = replace(&mut self.null_bitmap, new_bitmap_builder)
- .map(|mut builder| builder.finish())
- .unwrap();
-
- let old_bitmap = Bitmap::from(old_bitmap);
-
- for i in self.num_values..self.values_written {
- self.null_bitmap
- .as_mut()
- .unwrap()
- .append(old_bitmap.is_set(i));
- }
-
- Ok(Some(old_bitmap.into_buffer()))
- } else {
- Ok(None)
- }
- }
-
- /// Reset state of record reader.
- /// Should be called after consuming data, e.g. `consume_rep_levels`,
- /// `consume_rep_levels`, `consume_record_data` and `consume_bitmap_buffer`.
- pub fn reset(&mut self) {
- self.values_written -= self.num_values;
- self.num_records = 0;
- self.num_values = 0;
- self.values_seen = 0;
- self.in_middle_of_record = false;
- }
-
- /// Returns bitmap data.
- pub fn consume_bitmap(&mut self) -> Result<Option<Bitmap>> {
- self.consume_bitmap_buffer()
- .map(|buffer| buffer.map(Bitmap::from))
- }
-
- /// Try to read one batch of data.
- fn read_one_batch(&mut self, batch_size: usize) -> Result<usize> {
- // Reserve spaces
- self.records
- .resize(self.records.len() + batch_size * T::get_type_size(), 0);
- if let Some(ref mut buf) = self.rep_levels {
- buf.resize(buf.len() + batch_size * size_of::<i16>(), 0);
- }
- if let Some(ref mut buf) = self.def_levels {
- buf.resize(buf.len() + batch_size * size_of::<i16>(), 0);
- }
-
- let values_written = self.values_written;
-
- // Convert mutable buffer spaces to mutable slices
- let (prefix, values, suffix) =
- unsafe { self.records.as_slice_mut().align_to_mut::<T::T>() };
- assert!(prefix.is_empty() && suffix.is_empty());
- let values = &mut values[values_written..];
-
- let def_levels = self.def_levels.as_mut().map(|buf| {
- let (prefix, def_levels, suffix) =
- unsafe { buf.as_slice_mut().align_to_mut::<i16>() };
- assert!(prefix.is_empty() && suffix.is_empty());
- &mut def_levels[values_written..]
- });
-
- let rep_levels = self.rep_levels.as_mut().map(|buf| {
- let (prefix, rep_levels, suffix) =
- unsafe { buf.as_slice_mut().align_to_mut::<i16>() };
- assert!(prefix.is_empty() && suffix.is_empty());
- &mut rep_levels[values_written..]
- });
-
- let (values_read, levels_read) = self
- .column_reader
- .as_mut()
- .unwrap()
- .read_batch(batch_size, def_levels, rep_levels, values)?;
-
- // get new references for the def levels.
- let def_levels = self.def_levels.as_ref().map(|buf| {
- let (prefix, def_levels, suffix) =
- unsafe { buf.as_slice().align_to::<i16>() };
- assert!(prefix.is_empty() && suffix.is_empty());
- &def_levels[values_written..]
- });
-
- let max_def_level = self.column_desc.max_def_level();
-
- if values_read < levels_read {
- let def_levels = def_levels.ok_or_else(|| {
- general_err!(
- "Definition levels should exist when data is less than levels!"
- )
- })?;
-
- // Fill spaces in column data with default values
- let mut values_pos = values_read;
- let mut level_pos = levels_read;
-
- while level_pos > values_pos {
- if def_levels[level_pos - 1] == max_def_level {
- // This values is not empty
- // We use swap rather than assign here because T::T doesn't
- // implement Copy
- values.swap(level_pos - 1, values_pos - 1);
- values_pos -= 1;
- } else {
- values[level_pos - 1] = T::T::default();
- }
-
- level_pos -= 1;
- }
- }
-
- // Fill in bitmap data
- if let Some(null_buffer) = self.null_bitmap.as_mut() {
- let def_levels = def_levels.ok_or_else(|| {
- general_err!(
- "Definition levels should exist when data is less than levels!"
- )
- })?;
- (0..levels_read)
- .for_each(|idx| null_buffer.append(def_levels[idx] == max_def_level));
- }
-
- let values_read = max(values_read, levels_read);
- self.set_values_written(self.values_written + values_read)?;
- Ok(values_read)
- }
-
- /// Split values into records according repetition definition and returns number of
- /// records read.
- #[allow(clippy::unnecessary_wraps)]
- fn split_records(&mut self, records_to_read: usize) -> Result<usize> {
- let rep_levels = self.rep_levels.as_ref().map(|buf| {
- let (prefix, rep_levels, suffix) =
- unsafe { buf.as_slice().align_to::<i16>() };
- assert!(prefix.is_empty() && suffix.is_empty());
- rep_levels
- });
-
- match rep_levels {
- Some(buf) => {
- let mut records_read = 0;
-
- while (self.values_seen < self.values_written)
- && (records_read < records_to_read)
- {
- if buf[self.values_seen] == 0 {
- if self.in_middle_of_record {
- records_read += 1;
- self.num_records += 1;
- self.num_values = self.values_seen;
- }
- self.in_middle_of_record = true;
- }
- self.values_seen += 1;
- }
-
- Ok(records_read)
- }
- None => {
- let records_read =
- min(records_to_read, self.values_written - self.values_seen);
- self.num_records += records_read;
- self.num_values += records_read;
- self.values_seen += records_read;
- self.in_middle_of_record = false;
-
- Ok(records_read)
- }
- }
- }
-
- #[allow(clippy::unnecessary_wraps)]
- fn set_values_written(&mut self, new_values_written: usize) -> Result<()> {
- self.values_written = new_values_written;
- self.records
- .resize(self.values_written * T::get_type_size(), 0);
-
- let new_levels_len = self.values_written * size_of::<i16>();
-
- if let Some(ref mut buf) = self.rep_levels {
- buf.resize(new_levels_len, 0)
- };
-
- if let Some(ref mut buf) = self.def_levels {
- buf.resize(new_levels_len, 0)
- };
-
- Ok(())
- }
-}
-
-#[cfg(test)]
-mod tests {
- use super::RecordReader;
- use crate::basic::Encoding;
- use crate::column::page::Page;
- use crate::column::page::PageReader;
- use crate::data_type::Int32Type;
- use crate::errors::Result;
- use crate::schema::parser::parse_message_type;
- use crate::schema::types::SchemaDescriptor;
- use crate::util::test_common::page_util::{DataPageBuilder, DataPageBuilderImpl};
- use arrow::array::{BooleanBufferBuilder, Int16BufferBuilder, Int32BufferBuilder};
- use arrow::bitmap::Bitmap;
- use std::sync::Arc;
-
- struct TestPageReader {
- pages: Box<dyn Iterator<Item = Page>>,
- }
-
- impl TestPageReader {
- pub fn new(pages: Vec<Page>) -> Self {
- Self {
- pages: Box::new(pages.into_iter()),
- }
- }
- }
-
- impl PageReader for TestPageReader {
- fn get_next_page(&mut self) -> Result<Option<Page>> {
- Ok(self.pages.next())
- }
- }
-
- impl Iterator for TestPageReader {
- type Item = Result<Page>;
-
- fn next(&mut self) -> Option<Self::Item> {
- self.get_next_page().transpose()
- }
- }
-
- #[test]
- fn test_read_required_records() {
- // Construct column schema
- let message_type = "
- message test_schema {
- REQUIRED INT32 leaf;
- }
- ";
- let desc = parse_message_type(message_type)
- .map(|t| SchemaDescriptor::new(Arc::new(t)))
- .map(|s| s.column(0))
- .unwrap();
-
- // Construct record reader
- let mut record_reader = RecordReader::<Int32Type>::new(desc.clone());
-
- // First page
-
- // Records data:
- // test_schema
- // leaf: 4
- // test_schema
- // leaf: 7
- // test_schema
- // leaf: 6
- // test_schema
- // left: 3
- // test_schema
- // left: 2
- {
- let values = [4, 7, 6, 3, 2];
- let mut pb = DataPageBuilderImpl::new(desc.clone(), 5, true);
- pb.add_values::<Int32Type>(Encoding::PLAIN, &values);
- let page = pb.consume();
-
- let page_reader = Box::new(TestPageReader::new(vec![page]));
- record_reader.set_page_reader(page_reader).unwrap();
- assert_eq!(2, record_reader.read_records(2).unwrap());
- assert_eq!(2, record_reader.num_records());
- assert_eq!(2, record_reader.num_values());
- assert_eq!(3, record_reader.read_records(3).unwrap());
- assert_eq!(5, record_reader.num_records());
- assert_eq!(5, record_reader.num_values());
- }
-
- // Second page
-
- // Records data:
- // test_schema
- // leaf: 8
- // test_schema
- // leaf: 9
- {
- let values = [8, 9];
- let mut pb = DataPageBuilderImpl::new(desc, 2, true);
- pb.add_values::<Int32Type>(Encoding::PLAIN, &values);
- let page = pb.consume();
-
- let page_reader = Box::new(TestPageReader::new(vec![page]));
- record_reader.set_page_reader(page_reader).unwrap();
- assert_eq!(2, record_reader.read_records(10).unwrap());
- assert_eq!(7, record_reader.num_records());
- assert_eq!(7, record_reader.num_values());
- }
-
- let mut bb = Int32BufferBuilder::new(7);
- bb.append_slice(&[4, 7, 6, 3, 2, 8, 9]);
- let expected_buffer = bb.finish();
- assert_eq!(
- expected_buffer,
- record_reader.consume_record_data().unwrap()
- );
- assert_eq!(None, record_reader.consume_def_levels().unwrap());
- assert_eq!(None, record_reader.consume_bitmap().unwrap());
- }
-
- #[test]
- fn test_read_optional_records() {
- // Construct column schema
- let message_type = "
- message test_schema {
- OPTIONAL Group test_struct {
- OPTIONAL INT32 leaf;
- }
- }
- ";
-
- let desc = parse_message_type(message_type)
- .map(|t| SchemaDescriptor::new(Arc::new(t)))
- .map(|s| s.column(0))
- .unwrap();
-
- // Construct record reader
- let mut record_reader = RecordReader::<Int32Type>::new(desc.clone());
-
- // First page
-
- // Records data:
- // test_schema
- // test_struct
- // test_schema
- // test_struct
- // left: 7
- // test_schema
- // test_schema
- // test_struct
- // leaf: 6
- // test_schema
- // test_struct
- // leaf: 6
- {
- let values = [7, 6, 3];
- //empty, non-empty, empty, non-empty, non-empty
- let def_levels = [1i16, 2i16, 0i16, 2i16, 2i16];
- let mut pb = DataPageBuilderImpl::new(desc.clone(), 5, true);
- pb.add_def_levels(2, &def_levels);
- pb.add_values::<Int32Type>(Encoding::PLAIN, &values);
- let page = pb.consume();
-
- let page_reader = Box::new(TestPageReader::new(vec![page]));
- record_reader.set_page_reader(page_reader).unwrap();
- assert_eq!(2, record_reader.read_records(2).unwrap());
- assert_eq!(2, record_reader.num_records());
- assert_eq!(2, record_reader.num_values());
- assert_eq!(3, record_reader.read_records(3).unwrap());
- assert_eq!(5, record_reader.num_records());
- assert_eq!(5, record_reader.num_values());
- }
-
- // Second page
-
- // Records data:
- // test_schema
- // test_schema
- // test_struct
- // left: 8
- {
- let values = [8];
- //empty, non-empty
- let def_levels = [0i16, 2i16];
- let mut pb = DataPageBuilderImpl::new(desc, 2, true);
- pb.add_def_levels(2, &def_levels);
- pb.add_values::<Int32Type>(Encoding::PLAIN, &values);
- let page = pb.consume();
-
- let page_reader = Box::new(TestPageReader::new(vec![page]));
- record_reader.set_page_reader(page_reader).unwrap();
- assert_eq!(2, record_reader.read_records(10).unwrap());
- assert_eq!(7, record_reader.num_records());
- assert_eq!(7, record_reader.num_values());
- }
-
- // Verify result record data
- let mut bb = Int32BufferBuilder::new(7);
- bb.append_slice(&[0, 7, 0, 6, 3, 0, 8]);
- let expected_buffer = bb.finish();
- assert_eq!(
- expected_buffer,
- record_reader.consume_record_data().unwrap()
- );
-
- // Verify result def levels
- let mut bb = Int16BufferBuilder::new(7);
- bb.append_slice(&[1i16, 2i16, 0i16, 2i16, 2i16, 0i16, 2i16]);
- let expected_def_levels = bb.finish();
- assert_eq!(
- Some(expected_def_levels),
- record_reader.consume_def_levels().unwrap()
- );
-
- // Verify bitmap
- let mut bb = BooleanBufferBuilder::new(7);
- bb.append_slice(&[false, true, false, true, true, false, true]);
- let expected_bitmap = Bitmap::from(bb.finish());
- assert_eq!(
- Some(expected_bitmap),
- record_reader.consume_bitmap().unwrap()
- );
- }
-
- #[test]
- fn test_read_repeated_records() {
- // Construct column schema
- let message_type = "
- message test_schema {
- REPEATED Group test_struct {
- REPEATED INT32 leaf;
- }
- }
- ";
-
- let desc = parse_message_type(message_type)
- .map(|t| SchemaDescriptor::new(Arc::new(t)))
- .map(|s| s.column(0))
- .unwrap();
-
- // Construct record reader
- let mut record_reader = RecordReader::<Int32Type>::new(desc.clone());
-
- // First page
-
- // Records data:
- // test_schema
- // test_struct
- // leaf: 4
- // test_schema
- // test_schema
- // test_struct
- // test_struct
- // leaf: 7
- // leaf: 6
- // leaf: 3
- // test_struct
- // leaf: 2
- {
- let values = [4, 7, 6, 3, 2];
- let def_levels = [2i16, 0i16, 1i16, 2i16, 2i16, 2i16, 2i16];
- let rep_levels = [0i16, 0i16, 0i16, 1i16, 2i16, 2i16, 1i16];
- let mut pb = DataPageBuilderImpl::new(desc.clone(), 7, true);
- pb.add_rep_levels(2, &rep_levels);
- pb.add_def_levels(2, &def_levels);
- pb.add_values::<Int32Type>(Encoding::PLAIN, &values);
- let page = pb.consume();
-
- let page_reader = Box::new(TestPageReader::new(vec![page]));
- record_reader.set_page_reader(page_reader).unwrap();
-
- assert_eq!(1, record_reader.read_records(1).unwrap());
- assert_eq!(1, record_reader.num_records());
- assert_eq!(1, record_reader.num_values());
- assert_eq!(2, record_reader.read_records(3).unwrap());
- assert_eq!(3, record_reader.num_records());
- assert_eq!(7, record_reader.num_values());
- }
-
- // Second page
-
- // Records data:
- // test_schema
- // test_struct
- // leaf: 8
- // leaf: 9
- {
- let values = [8, 9];
- let def_levels = [2i16, 2i16];
- let rep_levels = [0i16, 2i16];
- let mut pb = DataPageBuilderImpl::new(desc, 2, true);
- pb.add_rep_levels(2, &rep_levels);
- pb.add_def_levels(2, &def_levels);
- pb.add_values::<Int32Type>(Encoding::PLAIN, &values);
- let page = pb.consume();
-
- let page_reader = Box::new(TestPageReader::new(vec![page]));
- record_reader.set_page_reader(page_reader).unwrap();
-
- assert_eq!(1, record_reader.read_records(10).unwrap());
- assert_eq!(4, record_reader.num_records());
- assert_eq!(9, record_reader.num_values());
- }
-
- // Verify result record data
- let mut bb = Int32BufferBuilder::new(9);
- bb.append_slice(&[4, 0, 0, 7, 6, 3, 2, 8, 9]);
- let expected_buffer = bb.finish();
- assert_eq!(
- expected_buffer,
- record_reader.consume_record_data().unwrap()
- );
-
- // Verify result def levels
- let mut bb = Int16BufferBuilder::new(9);
- bb.append_slice(&[2i16, 0i16, 1i16, 2i16, 2i16, 2i16, 2i16, 2i16, 2i16]);
- let expected_def_levels = bb.finish();
- assert_eq!(
- Some(expected_def_levels),
- record_reader.consume_def_levels().unwrap()
- );
-
- // Verify bitmap
- let mut bb = BooleanBufferBuilder::new(9);
- bb.append_slice(&[true, false, false, true, true, true, true, true, true]);
- let expected_bitmap = Bitmap::from(bb.finish());
- assert_eq!(
- Some(expected_bitmap),
- record_reader.consume_bitmap().unwrap()
- );
- }
-
- #[test]
- fn test_read_more_than_one_batch() {
- // Construct column schema
- let message_type = "
- message test_schema {
- REPEATED INT32 leaf;
- }
- ";
-
- let desc = parse_message_type(message_type)
- .map(|t| SchemaDescriptor::new(Arc::new(t)))
- .map(|s| s.column(0))
- .unwrap();
-
- // Construct record reader
- let mut record_reader = RecordReader::<Int32Type>::new(desc.clone());
-
- {
- let values = [100; 5000];
- let def_levels = [1i16; 5000];
- let mut rep_levels = [1i16; 5000];
- for idx in 0..1000 {
- rep_levels[idx * 5] = 0i16;
- }
-
- let mut pb = DataPageBuilderImpl::new(desc, 5000, true);
- pb.add_rep_levels(1, &rep_levels);
- pb.add_def_levels(1, &def_levels);
- pb.add_values::<Int32Type>(Encoding::PLAIN, &values);
- let page = pb.consume();
-
- let page_reader = Box::new(TestPageReader::new(vec![page]));
- record_reader.set_page_reader(page_reader).unwrap();
-
- assert_eq!(1000, record_reader.read_records(1000).unwrap());
- assert_eq!(1000, record_reader.num_records());
- assert_eq!(5000, record_reader.num_values());
- }
- }
-}
diff --git a/parquet/src/arrow/schema.rs b/parquet/src/arrow/schema.rs
deleted file mode 100644
index 6c04b70..0000000
--- a/parquet/src/arrow/schema.rs
+++ /dev/null
@@ -1,1943 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-//! Provides API for converting parquet schema to arrow schema and vice versa.
-//!
-//! The main interfaces for converting parquet schema to arrow schema are
-//! `parquet_to_arrow_schema`, `parquet_to_arrow_schema_by_columns` and
-//! `parquet_to_arrow_field`.
-//!
-//! The interfaces for converting arrow schema to parquet schema is coming.
-
-use std::collections::{HashMap, HashSet};
-use std::sync::Arc;
-
-use arrow::datatypes::{DataType, Field, IntervalUnit, Schema, TimeUnit};
-use arrow::ipc::writer;
-
-use crate::errors::{ParquetError::ArrowError, Result};
-use crate::file::{metadata::KeyValue, properties::WriterProperties};
-use crate::schema::types::{ColumnDescriptor, SchemaDescriptor, Type, TypePtr};
-use crate::{
- basic::{
- ConvertedType, DecimalType, IntType, LogicalType, Repetition, TimeType,
- TimeUnit as ParquetTimeUnit, TimestampType, Type as PhysicalType,
- },
- errors::ParquetError,
-};
-
-/// Convert Parquet schema to Arrow schema including optional metadata.
-/// Attempts to decode any existing Arrow schema metadata, falling back
-/// to converting the Parquet schema column-wise
-pub fn parquet_to_arrow_schema(
- parquet_schema: &SchemaDescriptor,
- key_value_metadata: &Option<Vec<KeyValue>>,
-) -> Result<Schema> {
- let mut metadata = parse_key_value_metadata(key_value_metadata).unwrap_or_default();
- metadata
- .remove(super::ARROW_SCHEMA_META_KEY)
- .map(|encoded| get_arrow_schema_from_metadata(&encoded))
- .unwrap_or(parquet_to_arrow_schema_by_columns(
- parquet_schema,
- 0..parquet_schema.columns().len(),
- key_value_metadata,
- ))
-}
-
-/// Convert parquet schema to arrow schema including optional metadata,
-/// only preserving some root columns.
-/// This is useful if we have columns `a.b`, `a.c.e` and `a.d`,
-/// and want `a` with all its child fields
-pub fn parquet_to_arrow_schema_by_root_columns<T>(
- parquet_schema: &SchemaDescriptor,
- column_indices: T,
- key_value_metadata: &Option<Vec<KeyValue>>,
-) -> Result<Schema>
-where
- T: IntoIterator<Item = usize>,
-{
- // Reconstruct the index ranges of the parent columns
- // An Arrow struct gets represented by 1+ columns based on how many child fields the
- // struct has. This means that getting fields 1 and 2 might return the struct twice,
- // if field 1 is the struct having say 3 fields, and field 2 is a primitive.
- //
- // The below gets the parent columns, and counts the number of child fields in each parent,
- // such that we would end up with:
- // - field 1 - columns: [0, 1, 2]
- // - field 2 - columns: [3]
- let mut parent_columns = vec![];
- let mut curr_name = "";
- let mut prev_name = "";
- let mut indices = vec![];
- (0..(parquet_schema.num_columns())).for_each(|i| {
- let p_type = parquet_schema.get_column_root(i);
- curr_name = p_type.get_basic_info().name();
- if prev_name.is_empty() {
- // first index
- indices.push(i);
- prev_name = curr_name;
- } else if curr_name != prev_name {
- prev_name = curr_name;
- parent_columns.push((curr_name.to_string(), indices.clone()));
- indices = vec![i];
- } else {
- indices.push(i);
- }
- });
- // push the last column if indices has values
- if !indices.is_empty() {
- parent_columns.push((curr_name.to_string(), indices));
- }
-
- // gather the required leaf columns
- let leaf_columns = column_indices
- .into_iter()
- .flat_map(|i| parent_columns[i].1.clone());
-
- parquet_to_arrow_schema_by_columns(parquet_schema, leaf_columns, key_value_metadata)
-}
-
-/// Convert parquet schema to arrow schema including optional metadata,
-/// only preserving some leaf columns.
-pub fn parquet_to_arrow_schema_by_columns<T>(
- parquet_schema: &SchemaDescriptor,
- column_indices: T,
- key_value_metadata: &Option<Vec<KeyValue>>,
-) -> Result<Schema>
-where
- T: IntoIterator<Item = usize>,
-{
- let mut metadata = parse_key_value_metadata(key_value_metadata).unwrap_or_default();
- let arrow_schema_metadata = metadata
- .remove(super::ARROW_SCHEMA_META_KEY)
- .map(|encoded| get_arrow_schema_from_metadata(&encoded))
- .map_or(Ok(None), |v| v.map(Some))?;
-
- // add the Arrow metadata to the Parquet metadata
- if let Some(arrow_schema) = &arrow_schema_metadata {
- arrow_schema.metadata().iter().for_each(|(k, v)| {
- metadata.insert(k.clone(), v.clone());
- });
- }
-
- let mut base_nodes = Vec::new();
- let mut base_nodes_set = HashSet::new();
- let mut leaves = HashSet::new();
-
- enum FieldType<'a> {
- Parquet(&'a Type),
- Arrow(Field),
- }
-
- for c in column_indices {
- let column = parquet_schema.column(c);
- let name = column.name();
-
- if let Some(field) = arrow_schema_metadata
- .as_ref()
- .and_then(|schema| schema.field_with_name(name).ok().cloned())
- {
- base_nodes.push(FieldType::Arrow(field));
- } else {
- let column = column.self_type() as *const Type;
- let root = parquet_schema.get_column_root(c);
- let root_raw_ptr = root as *const Type;
-
- leaves.insert(column);
- if !base_nodes_set.contains(&root_raw_ptr) {
- base_nodes.push(FieldType::Parquet(root));
- base_nodes_set.insert(root_raw_ptr);
- }
- }
- }
-
- base_nodes
- .into_iter()
- .map(|t| match t {
- FieldType::Parquet(t) => ParquetTypeConverter::new(t, &leaves).to_field(),
- FieldType::Arrow(f) => Ok(Some(f)),
- })
- .collect::<Result<Vec<Option<Field>>>>()
- .map(|result| result.into_iter().flatten().collect::<Vec<Field>>())
- .map(|fields| Schema::new_with_metadata(fields, metadata))
-}
-
-/// Try to convert Arrow schema metadata into a schema
-fn get_arrow_schema_from_metadata(encoded_meta: &str) -> Result<Schema> {
- let decoded = base64::decode(encoded_meta);
- match decoded {
- Ok(bytes) => {
- let slice = if bytes[0..4] == [255u8; 4] {
- &bytes[8..]
- } else {
- bytes.as_slice()
- };
- match arrow::ipc::root_as_message(slice) {
- Ok(message) => message
- .header_as_schema()
- .map(arrow::ipc::convert::fb_to_schema)
- .ok_or(ArrowError("the message is not Arrow Schema".to_string())),
- Err(err) => {
- // The flatbuffers implementation returns an error on verification error.
- Err(ArrowError(format!(
- "Unable to get root as message stored in {}: {:?}",
- super::ARROW_SCHEMA_META_KEY,
- err
- )))
- }
- }
- }
- Err(err) => {
- // The C++ implementation returns an error if the schema can't be parsed.
- Err(ArrowError(format!(
- "Unable to decode the encoded schema stored in {}, {:?}",
- super::ARROW_SCHEMA_META_KEY,
- err
- )))
- }
- }
-}
-
-/// Encodes the Arrow schema into the IPC format, and base64 encodes it
-fn encode_arrow_schema(schema: &Schema) -> String {
- let options = writer::IpcWriteOptions::default();
- let data_gen = arrow::ipc::writer::IpcDataGenerator::default();
- let mut serialized_schema = data_gen.schema_to_bytes(&schema, &options);
-
- // manually prepending the length to the schema as arrow uses the legacy IPC format
- // TODO: change after addressing ARROW-9777
- let schema_len = serialized_schema.ipc_message.len();
- let mut len_prefix_schema = Vec::with_capacity(schema_len + 8);
- len_prefix_schema.append(&mut vec![255u8, 255, 255, 255]);
- len_prefix_schema.append((schema_len as u32).to_le_bytes().to_vec().as_mut());
- len_prefix_schema.append(&mut serialized_schema.ipc_message);
-
- base64::encode(&len_prefix_schema)
-}
-
-/// Mutates writer metadata by storing the encoded Arrow schema.
-/// If there is an existing Arrow schema metadata, it is replaced.
-pub(crate) fn add_encoded_arrow_schema_to_metadata(
- schema: &Schema,
- props: &mut WriterProperties,
-) {
- let encoded = encode_arrow_schema(schema);
-
- let schema_kv = KeyValue {
- key: super::ARROW_SCHEMA_META_KEY.to_string(),
- value: Some(encoded),
- };
-
- let mut meta = props.key_value_metadata.clone().unwrap_or_default();
- // check if ARROW:schema exists, and overwrite it
- let schema_meta = meta
- .iter()
- .enumerate()
- .find(|(_, kv)| kv.key.as_str() == super::ARROW_SCHEMA_META_KEY);
- match schema_meta {
- Some((i, _)) => {
- meta.remove(i);
- meta.push(schema_kv);
- }
- None => {
- meta.push(schema_kv);
- }
- }
- props.key_value_metadata = Some(meta);
-}
-
-/// Convert arrow schema to parquet schema
-pub fn arrow_to_parquet_schema(schema: &Schema) -> Result<SchemaDescriptor> {
- let fields: Result<Vec<TypePtr>> = schema
- .fields()
- .iter()
- .map(|field| arrow_to_parquet_type(field).map(Arc::new))
- .collect();
- let group = Type::group_type_builder("arrow_schema")
- .with_fields(&mut fields?)
- .build()?;
- Ok(SchemaDescriptor::new(Arc::new(group)))
-}
-
-fn parse_key_value_metadata(
- key_value_metadata: &Option<Vec<KeyValue>>,
-) -> Option<HashMap<String, String>> {
- match key_value_metadata {
- Some(key_values) => {
- let map: HashMap<String, String> = key_values
- .iter()
- .filter_map(|kv| {
- kv.value
- .as_ref()
- .map(|value| (kv.key.clone(), value.clone()))
- })
- .collect();
-
- if map.is_empty() {
- None
- } else {
- Some(map)
- }
- }
- None => None,
- }
-}
-
-/// Convert parquet column schema to arrow field.
-pub fn parquet_to_arrow_field(parquet_column: &ColumnDescriptor) -> Result<Field> {
- let schema = parquet_column.self_type();
-
- let mut leaves = HashSet::new();
- leaves.insert(parquet_column.self_type() as *const Type);
-
- ParquetTypeConverter::new(schema, &leaves)
- .to_field()
- .map(|opt| opt.unwrap())
-}
-
-pub fn decimal_length_from_precision(precision: usize) -> usize {
- (10.0_f64.powi(precision as i32).log2() / 8.0).ceil() as usize
-}
-
-/// Convert an arrow field to a parquet `Type`
-fn arrow_to_parquet_type(field: &Field) -> Result<Type> {
- let name = field.name().as_str();
- let repetition = if field.is_nullable() {
- Repetition::OPTIONAL
- } else {
- Repetition::REQUIRED
- };
- // create type from field
- match field.data_type() {
- DataType::Null => Type::primitive_type_builder(name, PhysicalType::INT32)
- .with_logical_type(Some(LogicalType::UNKNOWN(Default::default())))
- .with_repetition(repetition)
- .build(),
- DataType::Boolean => Type::primitive_type_builder(name, PhysicalType::BOOLEAN)
- .with_repetition(repetition)
- .build(),
- DataType::Int8 => Type::primitive_type_builder(name, PhysicalType::INT32)
- .with_logical_type(Some(LogicalType::INTEGER(IntType {
- bit_width: 8,
- is_signed: true,
- })))
- .with_repetition(repetition)
- .build(),
- DataType::Int16 => Type::primitive_type_builder(name, PhysicalType::INT32)
- .with_logical_type(Some(LogicalType::INTEGER(IntType {
- bit_width: 16,
- is_signed: true,
- })))
- .with_repetition(repetition)
- .build(),
- DataType::Int32 => Type::primitive_type_builder(name, PhysicalType::INT32)
- .with_repetition(repetition)
- .build(),
- DataType::Int64 => Type::primitive_type_builder(name, PhysicalType::INT64)
- .with_repetition(repetition)
- .build(),
- DataType::UInt8 => Type::primitive_type_builder(name, PhysicalType::INT32)
- .with_logical_type(Some(LogicalType::INTEGER(IntType {
- bit_width: 8,
- is_signed: false,
- })))
- .with_repetition(repetition)
- .build(),
- DataType::UInt16 => Type::primitive_type_builder(name, PhysicalType::INT32)
- .with_logical_type(Some(LogicalType::INTEGER(IntType {
- bit_width: 16,
- is_signed: false,
- })))
- .with_repetition(repetition)
- .build(),
- DataType::UInt32 => Type::primitive_type_builder(name, PhysicalType::INT32)
- .with_logical_type(Some(LogicalType::INTEGER(IntType {
- bit_width: 32,
- is_signed: false,
- })))
- .with_repetition(repetition)
- .build(),
- DataType::UInt64 => Type::primitive_type_builder(name, PhysicalType::INT64)
- .with_logical_type(Some(LogicalType::INTEGER(IntType {
- bit_width: 64,
- is_signed: false,
- })))
- .with_repetition(repetition)
- .build(),
- DataType::Float16 => Err(ArrowError("Float16 arrays not supported".to_string())),
- DataType::Float32 => Type::primitive_type_builder(name, PhysicalType::FLOAT)
- .with_repetition(repetition)
- .build(),
- DataType::Float64 => Type::primitive_type_builder(name, PhysicalType::DOUBLE)
- .with_repetition(repetition)
- .build(),
- DataType::Timestamp(time_unit, zone) => Type::primitive_type_builder(
- name,
- PhysicalType::INT64,
- )
- .with_logical_type(Some(LogicalType::TIMESTAMP(TimestampType {
- is_adjusted_to_u_t_c: matches!(zone, Some(z) if !z.as_str().is_empty()),
- unit: match time_unit {
- TimeUnit::Second => ParquetTimeUnit::MILLIS(Default::default()),
- TimeUnit::Millisecond => ParquetTimeUnit::MILLIS(Default::default()),
- TimeUnit::Microsecond => ParquetTimeUnit::MICROS(Default::default()),
- TimeUnit::Nanosecond => ParquetTimeUnit::NANOS(Default::default()),
- },
- })))
- .with_repetition(repetition)
- .build(),
- DataType::Date32 => Type::primitive_type_builder(name, PhysicalType::INT32)
- .with_logical_type(Some(LogicalType::DATE(Default::default())))
- .with_repetition(repetition)
- .build(),
- // date64 is cast to date32
- DataType::Date64 => Type::primitive_type_builder(name, PhysicalType::INT32)
- .with_logical_type(Some(LogicalType::DATE(Default::default())))
- .with_repetition(repetition)
- .build(),
- DataType::Time32(_) => Type::primitive_type_builder(name, PhysicalType::INT32)
- .with_logical_type(Some(LogicalType::TIME(TimeType {
- is_adjusted_to_u_t_c: false,
- unit: ParquetTimeUnit::MILLIS(Default::default()),
- })))
- .with_repetition(repetition)
- .build(),
- DataType::Time64(unit) => Type::primitive_type_builder(name, PhysicalType::INT64)
- .with_logical_type(Some(LogicalType::TIME(TimeType {
- is_adjusted_to_u_t_c: false,
- unit: match unit {
- TimeUnit::Microsecond => ParquetTimeUnit::MICROS(Default::default()),
- TimeUnit::Nanosecond => ParquetTimeUnit::NANOS(Default::default()),
- u => unreachable!("Invalid unit for Time64: {:?}", u),
- },
- })))
- .with_repetition(repetition)
- .build(),
- DataType::Duration(_) => Err(ArrowError(
- "Converting Duration to parquet not supported".to_string(),
- )),
- DataType::Interval(_) => {
- Type::primitive_type_builder(name, PhysicalType::FIXED_LEN_BYTE_ARRAY)
- .with_converted_type(ConvertedType::INTERVAL)
- .with_repetition(repetition)
- .with_length(12)
- .build()
- }
- DataType::Binary | DataType::LargeBinary => {
- Type::primitive_type_builder(name, PhysicalType::BYTE_ARRAY)
- .with_repetition(repetition)
- .build()
- }
- DataType::FixedSizeBinary(length) => {
- Type::primitive_type_builder(name, PhysicalType::FIXED_LEN_BYTE_ARRAY)
- .with_repetition(repetition)
- .with_length(*length)
- .build()
- }
- DataType::Decimal(precision, scale) => {
- // Decimal precision determines the Parquet physical type to use.
- // TODO(ARROW-12018): Enable the below after ARROW-10818 Decimal support
- //
- // let (physical_type, length) = if *precision > 1 && *precision <= 9 {
- // (PhysicalType::INT32, -1)
- // } else if *precision <= 18 {
- // (PhysicalType::INT64, -1)
- // } else {
- // (
- // PhysicalType::FIXED_LEN_BYTE_ARRAY,
- // decimal_length_from_precision(*precision) as i32,
- // )
- // };
- Type::primitive_type_builder(name, PhysicalType::FIXED_LEN_BYTE_ARRAY)
- .with_repetition(repetition)
- .with_length(decimal_length_from_precision(*precision) as i32)
- .with_logical_type(Some(LogicalType::DECIMAL(DecimalType {
- scale: *scale as i32,
- precision: *precision as i32,
- })))
- .with_precision(*precision as i32)
- .with_scale(*scale as i32)
- .build()
- }
- DataType::Utf8 | DataType::LargeUtf8 => {
- Type::primitive_type_builder(name, PhysicalType::BYTE_ARRAY)
- .with_logical_type(Some(LogicalType::STRING(Default::default())))
- .with_repetition(repetition)
- .build()
- }
- DataType::List(f) | DataType::FixedSizeList(f, _) | DataType::LargeList(f) => {
- Type::group_type_builder(name)
- .with_fields(&mut vec![Arc::new(
- Type::group_type_builder("list")
- .with_fields(&mut vec![Arc::new(arrow_to_parquet_type(f)?)])
- .with_repetition(Repetition::REPEATED)
- .build()?,
- )])
- .with_logical_type(Some(LogicalType::LIST(Default::default())))
- .with_repetition(repetition)
- .build()
- }
- DataType::Struct(fields) => {
- if fields.is_empty() {
- return Err(ArrowError(
- "Parquet does not support writing empty structs".to_string(),
- ));
- }
- // recursively convert children to types/nodes
- let fields: Result<Vec<TypePtr>> = fields
- .iter()
- .map(|f| arrow_to_parquet_type(f).map(Arc::new))
- .collect();
- Type::group_type_builder(name)
- .with_fields(&mut fields?)
- .with_repetition(repetition)
- .build()
- }
- DataType::Union(_) => unimplemented!("See ARROW-8817."),
- DataType::Dictionary(_, ref value) => {
- // Dictionary encoding not handled at the schema level
- let dict_field = Field::new(name, *value.clone(), field.is_nullable());
- arrow_to_parquet_type(&dict_field)
- }
- }
-}
-/// This struct is used to group methods and data structures used to convert parquet
-/// schema together.
-struct ParquetTypeConverter<'a> {
- schema: &'a Type,
- /// This is the columns that need to be converted to arrow schema.
- columns_to_convert: &'a HashSet<*const Type>,
-}
-
-impl<'a> ParquetTypeConverter<'a> {
- fn new(schema: &'a Type, columns_to_convert: &'a HashSet<*const Type>) -> Self {
- Self {
- schema,
- columns_to_convert,
- }
- }
-
- fn clone_with_schema(&self, other: &'a Type) -> Self {
- Self {
- schema: other,
- columns_to_convert: self.columns_to_convert,
- }
- }
-}
-
-impl ParquetTypeConverter<'_> {
- // Public interfaces.
-
- /// Converts parquet schema to arrow data type.
- ///
- /// This function discards schema name.
- ///
- /// If this schema is a primitive type and not included in the leaves, the result is
- /// Ok(None).
- ///
- /// If this schema is a group type and none of its children is reserved in the
- /// conversion, the result is Ok(None).
- fn to_data_type(&self) -> Result<Option<DataType>> {
- match self.schema {
- Type::PrimitiveType { .. } => self.to_primitive_type(),
- Type::GroupType { .. } => self.to_group_type(),
- }
- }
-
- /// Converts parquet schema to arrow field.
- ///
- /// This method is roughly the same as
- /// [`to_data_type`](`ParquetTypeConverter::to_data_type`), except it reserves schema
- /// name.
- fn to_field(&self) -> Result<Option<Field>> {
- self.to_data_type().map(|opt| {
- opt.map(|dt| Field::new(self.schema.name(), dt, self.is_nullable()))
- })
- }
-
- // Utility functions.
-
- /// Checks whether this schema is nullable.
- fn is_nullable(&self) -> bool {
- let basic_info = self.schema.get_basic_info();
- if basic_info.has_repetition() {
- match basic_info.repetition() {
- Repetition::OPTIONAL => true,
- Repetition::REPEATED => true,
- Repetition::REQUIRED => false,
- }
- } else {
- false
- }
- }
-
- fn is_repeated(&self) -> bool {
- let basic_info = self.schema.get_basic_info();
-
- basic_info.has_repetition() && basic_info.repetition() == Repetition::REPEATED
- }
-
- fn is_self_included(&self) -> bool {
- self.columns_to_convert
- .contains(&(self.schema as *const Type))
- }
-
- // Functions for primitive types.
-
- /// Entry point for converting parquet primitive type to arrow type.
- ///
- /// This function takes care of repetition.
- fn to_primitive_type(&self) -> Result<Option<DataType>> {
- if self.is_self_included() {
- self.to_primitive_type_inner().map(|dt| {
- if self.is_repeated() {
- Some(DataType::List(Box::new(Field::new(
- self.schema.name(),
- dt,
- self.is_nullable(),
- ))))
- } else {
- Some(dt)
- }
- })
- } else {
- Ok(None)
- }
- }
-
- /// Converting parquet primitive type to arrow data type.
- fn to_primitive_type_inner(&self) -> Result<DataType> {
- match self.schema.get_physical_type() {
- PhysicalType::BOOLEAN => Ok(DataType::Boolean),
- PhysicalType::INT32 => self.from_int32(),
- PhysicalType::INT64 => self.from_int64(),
- PhysicalType::INT96 => Ok(DataType::Timestamp(TimeUnit::Nanosecond, None)),
- PhysicalType::FLOAT => Ok(DataType::Float32),
- PhysicalType::DOUBLE => Ok(DataType::Float64),
- PhysicalType::BYTE_ARRAY => self.from_byte_array(),
- PhysicalType::FIXED_LEN_BYTE_ARRAY => self.from_fixed_len_byte_array(),
- }
- }
-
- fn from_int32(&self) -> Result<DataType> {
- match (
- self.schema.get_basic_info().logical_type(),
- self.schema.get_basic_info().converted_type(),
- ) {
- (None, ConvertedType::NONE) => Ok(DataType::Int32),
- (Some(LogicalType::INTEGER(t)), _) => match (t.bit_width, t.is_signed) {
- (8, true) => Ok(DataType::Int8),
- (16, true) => Ok(DataType::Int16),
- (32, true) => Ok(DataType::Int32),
- (8, false) => Ok(DataType::UInt8),
- (16, false) => Ok(DataType::UInt16),
- (32, false) => Ok(DataType::UInt32),
- _ => Err(ArrowError(format!(
- "Cannot create INT32 physical type from {:?}",
- t
- ))),
- },
- (Some(LogicalType::DECIMAL(_)), _) => Ok(self.to_decimal()),
- (Some(LogicalType::DATE(_)), _) => Ok(DataType::Date32),
- (Some(LogicalType::TIME(t)), _) => match t.unit {
- ParquetTimeUnit::MILLIS(_) => Ok(DataType::Time32(TimeUnit::Millisecond)),
- _ => Err(ArrowError(format!(
- "Cannot create INT32 physical type from {:?}",
- t.unit
- ))),
- },
- (None, ConvertedType::UINT_8) => Ok(DataType::UInt8),
- (None, ConvertedType::UINT_16) => Ok(DataType::UInt16),
- (None, ConvertedType::UINT_32) => Ok(DataType::UInt32),
- (None, ConvertedType::INT_8) => Ok(DataType::Int8),
- (None, ConvertedType::INT_16) => Ok(DataType::Int16),
- (None, ConvertedType::INT_32) => Ok(DataType::Int32),
- (None, ConvertedType::DATE) => Ok(DataType::Date32),
- (None, ConvertedType::TIME_MILLIS) => {
- Ok(DataType::Time32(TimeUnit::Millisecond))
- }
- (None, ConvertedType::DECIMAL) => Ok(self.to_decimal()),
- (logical, converted) => Err(ArrowError(format!(
- "Unable to convert parquet INT32 logical type {:?} or converted type {}",
- logical, converted
- ))),
- }
- }
-
- fn from_int64(&self) -> Result<DataType> {
- match (
- self.schema.get_basic_info().logical_type(),
- self.schema.get_basic_info().converted_type(),
- ) {
- (None, ConvertedType::NONE) => Ok(DataType::Int64),
- (Some(LogicalType::INTEGER(t)), _) if t.bit_width == 64 => {
- match t.is_signed {
- true => Ok(DataType::Int64),
- false => Ok(DataType::UInt64),
- }
- }
- (Some(LogicalType::TIME(t)), _) => match t.unit {
- ParquetTimeUnit::MILLIS(_) => Err(ArrowError(
- "Cannot create INT64 from MILLIS time unit".to_string(),
- )),
- ParquetTimeUnit::MICROS(_) => Ok(DataType::Time64(TimeUnit::Microsecond)),
- ParquetTimeUnit::NANOS(_) => Ok(DataType::Time64(TimeUnit::Nanosecond)),
- },
- (Some(LogicalType::TIMESTAMP(t)), _) => Ok(DataType::Timestamp(
- match t.unit {
- ParquetTimeUnit::MILLIS(_) => TimeUnit::Millisecond,
- ParquetTimeUnit::MICROS(_) => TimeUnit::Microsecond,
- ParquetTimeUnit::NANOS(_) => TimeUnit::Nanosecond,
- },
- if t.is_adjusted_to_u_t_c {
- Some("UTC".to_string())
- } else {
- None
- },
- )),
- (None, ConvertedType::INT_64) => Ok(DataType::Int64),
- (None, ConvertedType::UINT_64) => Ok(DataType::UInt64),
- (None, ConvertedType::TIME_MICROS) => {
- Ok(DataType::Time64(TimeUnit::Microsecond))
- }
- (None, ConvertedType::TIMESTAMP_MILLIS) => {
- Ok(DataType::Timestamp(TimeUnit::Millisecond, None))
- }
- (None, ConvertedType::TIMESTAMP_MICROS) => {
- Ok(DataType::Timestamp(TimeUnit::Microsecond, None))
- }
- (Some(LogicalType::DECIMAL(_)), _) => Ok(self.to_decimal()),
- (None, ConvertedType::DECIMAL) => Ok(self.to_decimal()),
- (logical, converted) => Err(ArrowError(format!(
- "Unable to convert parquet INT64 logical type {:?} or converted type {}",
- logical, converted
- ))),
- }
- }
-
- fn from_fixed_len_byte_array(&self) -> Result<DataType> {
- match (
- self.schema.get_basic_info().logical_type(),
- self.schema.get_basic_info().converted_type(),
- ) {
- (Some(LogicalType::DECIMAL(_)), _) => Ok(self.to_decimal()),
- (None, ConvertedType::DECIMAL) => Ok(self.to_decimal()),
- (None, ConvertedType::INTERVAL) => {
- // There is currently no reliable way of determining which IntervalUnit
- // to return. Thus without the original Arrow schema, the results
- // would be incorrect if all 12 bytes of the interval are populated
- Ok(DataType::Interval(IntervalUnit::DayTime))
- }
- _ => {
- let byte_width = match self.schema {
- Type::PrimitiveType {
- ref type_length, ..
- } => *type_length,
- _ => {
- return Err(ArrowError(
- "Expected a physical type, not a group type".to_string(),
- ))
- }
- };
-
- Ok(DataType::FixedSizeBinary(byte_width))
- }
- }
- }
-
- fn to_decimal(&self) -> DataType {
- assert!(self.schema.is_primitive());
- DataType::Decimal(
- self.schema.get_precision() as usize,
- self.schema.get_scale() as usize,
- )
- }
-
- fn from_byte_array(&self) -> Result<DataType> {
- match (self.schema.get_basic_info().logical_type(), self.schema.get_basic_info().converted_type()) {
- (Some(LogicalType::STRING(_)), _) => Ok(DataType::Utf8),
- (Some(LogicalType::JSON(_)), _) => Ok(DataType::Binary),
- (Some(LogicalType::BSON(_)), _) => Ok(DataType::Binary),
- (Some(LogicalType::ENUM(_)), _) => Ok(DataType::Binary),
- (None, ConvertedType::NONE) => Ok(DataType::Binary),
- (None, ConvertedType::JSON) => Ok(DataType::Binary),
- (None, ConvertedType::BSON) => Ok(DataType::Binary),
- (None, ConvertedType::ENUM) => Ok(DataType::Binary),
- (None, ConvertedType::UTF8) => Ok(DataType::Utf8),
- (logical, converted) => Err(ArrowError(format!(
- "Unable to convert parquet BYTE_ARRAY logical type {:?} or converted type {}",
- logical, converted
- ))),
- }
- }
-
- // Functions for group types.
-
- /// Entry point for converting parquet group type.
- ///
- /// This function takes care of logical type and repetition.
- fn to_group_type(&self) -> Result<Option<DataType>> {
- if self.is_repeated() {
- self.to_struct().map(|opt| {
- opt.map(|dt| {
- DataType::List(Box::new(Field::new(
- self.schema.name(),
- dt,
- self.is_nullable(),
- )))
- })
- })
- } else {
- match (
- self.schema.get_basic_info().logical_type(),
- self.schema.get_basic_info().converted_type(),
- ) {
- (Some(LogicalType::LIST(_)), _) => self.to_list(),
- (None, ConvertedType::LIST) => self.to_list(),
- _ => self.to_struct(),
- }
- }
- }
-
- /// Converts a parquet group type to arrow struct.
- fn to_struct(&self) -> Result<Option<DataType>> {
- match self.schema {
- Type::PrimitiveType { .. } => Err(ParquetError::General(format!(
- "{:?} is a struct type, and can't be processed as primitive.",
- self.schema
- ))),
- Type::GroupType {
- basic_info: _,
- fields,
- } => fields
- .iter()
- .map(|field_ptr| self.clone_with_schema(field_ptr).to_field())
- .collect::<Result<Vec<Option<Field>>>>()
- .map(|result| result.into_iter().flatten().collect::<Vec<Field>>())
- .map(|fields| {
- if fields.is_empty() {
- None
- } else {
- Some(DataType::Struct(fields))
- }
- }),
- }
- }
-
- /// Converts a parquet list to arrow list.
- ///
- /// To fully understand this algorithm, please refer to
- /// [parquet doc](https://github.com/apache/parquet-format/blob/master/LogicalTypes.md).
- fn to_list(&self) -> Result<Option<DataType>> {
- match self.schema {
- Type::PrimitiveType { .. } => Err(ParquetError::General(format!(
- "{:?} is a list type and can't be processed as primitive.",
- self.schema
- ))),
- Type::GroupType {
- basic_info: _,
- fields,
- } if fields.len() == 1 => {
- let list_item = fields.first().unwrap();
- let item_converter = self.clone_with_schema(list_item);
-
- let item_type = match list_item.as_ref() {
- Type::PrimitiveType { .. } => {
- if item_converter.is_repeated() {
- item_converter.to_primitive_type_inner().map(Some)
- } else {
- Err(ArrowError(
- "Primitive element type of list must be repeated."
- .to_string(),
- ))
- }
- }
- Type::GroupType {
- basic_info: _,
- fields,
- } => {
- if fields.len() > 1 {
- item_converter.to_struct()
- } else if fields.len() == 1
- && list_item.name() != "array"
- && list_item.name() != format!("{}_tuple", self.schema.name())
- {
- let nested_item = fields.first().unwrap();
- let nested_item_converter =
- self.clone_with_schema(nested_item);
-
- nested_item_converter.to_data_type()
- } else {
- item_converter.to_struct()
- }
- }
- };
-
- // Check that the name of the list child is "list", in which case we
- // get the child nullability and name (normally "element") from the nested
- // group type.
- // Without this step, the child incorrectly inherits the parent's optionality
- let (list_item_name, item_is_optional) = match &item_converter.schema {
- Type::GroupType { basic_info, fields }
- if basic_info.name() == "list" && fields.len() == 1 =>
- {
- let field = fields.first().unwrap();
- (field.name(), field.is_optional())
- }
- _ => (list_item.name(), list_item.is_optional()),
- };
-
- item_type.map(|opt| {
- opt.map(|dt| {
- DataType::List(Box::new(Field::new(
- list_item_name,
- dt,
- item_is_optional,
- )))
- })
- })
- }
- _ => Err(ArrowError(
- "Group element type of list can only contain one field.".to_string(),
- )),
- }
- }
-}
-
-#[cfg(test)]
-mod tests {
- use super::*;
-
- use std::{collections::HashMap, convert::TryFrom, sync::Arc};
-
- use arrow::datatypes::{DataType, Field, IntervalUnit, TimeUnit};
-
- use crate::file::{metadata::KeyValue, reader::SerializedFileReader};
- use crate::{
- arrow::{ArrowReader, ArrowWriter, ParquetFileArrowReader},
- schema::{parser::parse_message_type, types::SchemaDescriptor},
- util::test_common::get_temp_file,
- };
-
- #[test]
- fn test_flat_primitives() {
- let message_type = "
- message test_schema {
- REQUIRED BOOLEAN boolean;
- REQUIRED INT32 int8 (INT_8);
- REQUIRED INT32 int16 (INT_16);
- REQUIRED INT32 uint8 (INTEGER(8,false));
- REQUIRED INT32 uint16 (INTEGER(16,false));
- REQUIRED INT32 int32;
- REQUIRED INT64 int64 ;
- OPTIONAL DOUBLE double;
- OPTIONAL FLOAT float;
- OPTIONAL BINARY string (UTF8);
- OPTIONAL BINARY string_2 (STRING);
- }
- ";
- let parquet_group_type = parse_message_type(message_type).unwrap();
-
- let parquet_schema = SchemaDescriptor::new(Arc::new(parquet_group_type));
- let converted_arrow_schema =
- parquet_to_arrow_schema(&parquet_schema, &None).unwrap();
-
- let arrow_fields = vec![
- Field::new("boolean", DataType::Boolean, false),
- Field::new("int8", DataType::Int8, false),
- Field::new("int16", DataType::Int16, false),
- Field::new("uint8", DataType::UInt8, false),
- Field::new("uint16", DataType::UInt16, false),
- Field::new("int32", DataType::Int32, false),
- Field::new("int64", DataType::Int64, false),
- Field::new("double", DataType::Float64, true),
- Field::new("float", DataType::Float32, true),
- Field::new("string", DataType::Utf8, true),
- Field::new("string_2", DataType::Utf8, true),
- ];
-
- assert_eq!(&arrow_fields, converted_arrow_schema.fields());
- }
-
- #[test]
- fn test_byte_array_fields() {
- let message_type = "
- message test_schema {
- REQUIRED BYTE_ARRAY binary;
- REQUIRED FIXED_LEN_BYTE_ARRAY (20) fixed_binary;
- }
- ";
-
- let parquet_group_type = parse_message_type(message_type).unwrap();
-
- let parquet_schema = SchemaDescriptor::new(Arc::new(parquet_group_type));
- let converted_arrow_schema =
- parquet_to_arrow_schema(&parquet_schema, &None).unwrap();
-
- let arrow_fields = vec![
- Field::new("binary", DataType::Binary, false),
- Field::new("fixed_binary", DataType::FixedSizeBinary(20), false),
- ];
- assert_eq!(&arrow_fields, converted_arrow_schema.fields());
- }
-
- #[test]
- fn test_duplicate_fields() {
- let message_type = "
- message test_schema {
- REQUIRED BOOLEAN boolean;
- REQUIRED INT32 int8 (INT_8);
- }
- ";
-
- let parquet_group_type = parse_message_type(message_type).unwrap();
-
- let parquet_schema = SchemaDescriptor::new(Arc::new(parquet_group_type));
- let converted_arrow_schema =
- parquet_to_arrow_schema(&parquet_schema, &None).unwrap();
-
- let arrow_fields = vec![
- Field::new("boolean", DataType::Boolean, false),
- Field::new("int8", DataType::Int8, false),
- ];
- assert_eq!(&arrow_fields, converted_arrow_schema.fields());
-
- let converted_arrow_schema = parquet_to_arrow_schema_by_columns(
- &parquet_schema,
- vec![0usize, 1usize],
- &None,
- )
- .unwrap();
- assert_eq!(&arrow_fields, converted_arrow_schema.fields());
- }
-
- #[test]
- fn test_parquet_lists() {
- let mut arrow_fields = Vec::new();
-
- // LIST encoding example taken from parquet-format/LogicalTypes.md
- let message_type = "
- message test_schema {
- REQUIRED GROUP my_list (LIST) {
- REPEATED GROUP list {
- OPTIONAL BINARY element (UTF8);
- }
- }
- OPTIONAL GROUP my_list (LIST) {
- REPEATED GROUP list {
- REQUIRED BINARY element (UTF8);
- }
- }
- OPTIONAL GROUP array_of_arrays (LIST) {
- REPEATED GROUP list {
- REQUIRED GROUP element (LIST) {
- REPEATED GROUP list {
- REQUIRED INT32 element;
- }
- }
- }
- }
- OPTIONAL GROUP my_list (LIST) {
- REPEATED GROUP element {
- REQUIRED BINARY str (UTF8);
- }
- }
- OPTIONAL GROUP my_list (LIST) {
- REPEATED INT32 element;
- }
- OPTIONAL GROUP my_list (LIST) {
- REPEATED GROUP element {
- REQUIRED BINARY str (UTF8);
- REQUIRED INT32 num;
- }
- }
- OPTIONAL GROUP my_list (LIST) {
- REPEATED GROUP array {
- REQUIRED BINARY str (UTF8);
- }
-
- }
- OPTIONAL GROUP my_list (LIST) {
- REPEATED GROUP my_list_tuple {
- REQUIRED BINARY str (UTF8);
- }
- }
- REPEATED INT32 name;
- }
- ";
-
- // // List<String> (list non-null, elements nullable)
- // required group my_list (LIST) {
- // repeated group list {
- // optional binary element (UTF8);
- // }
- // }
- {
- arrow_fields.push(Field::new(
- "my_list",
- DataType::List(Box::new(Field::new("element", DataType::Utf8, true))),
- false,
- ));
- }
-
- // // List<String> (list nullable, elements non-null)
- // optional group my_list (LIST) {
- // repeated group list {
- // required binary element (UTF8);
- // }
- // }
- {
- arrow_fields.push(Field::new(
- "my_list",
- DataType::List(Box::new(Field::new("element", DataType::Utf8, false))),
- true,
- ));
- }
-
- // Element types can be nested structures. For example, a list of lists:
- //
- // // List<List<Integer>>
- // optional group array_of_arrays (LIST) {
- // repeated group list {
- // required group element (LIST) {
- // repeated group list {
- // required int32 element;
- // }
- // }
- // }
- // }
- {
- let arrow_inner_list =
- DataType::List(Box::new(Field::new("element", DataType::Int32, false)));
- arrow_fields.push(Field::new(
- "array_of_arrays",
- DataType::List(Box::new(Field::new("element", arrow_inner_list, false))),
- true,
- ));
- }
-
- // // List<String> (list nullable, elements non-null)
- // optional group my_list (LIST) {
- // repeated group element {
- // required binary str (UTF8);
- // };
- // }
- {
- arrow_fields.push(Field::new(
- "my_list",
- DataType::List(Box::new(Field::new("element", DataType::Utf8, true))),
- true,
- ));
- }
-
- // // List<Integer> (nullable list, non-null elements)
- // optional group my_list (LIST) {
- // repeated int32 element;
- // }
- {
- arrow_fields.push(Field::new(
- "my_list",
- DataType::List(Box::new(Field::new("element", DataType::Int32, true))),
- true,
- ));
- }
-
- // // List<Tuple<String, Integer>> (nullable list, non-null elements)
- // optional group my_list (LIST) {
- // repeated group element {
- // required binary str (UTF8);
- // required int32 num;
- // };
- // }
- {
- let arrow_struct = DataType::Struct(vec![
- Field::new("str", DataType::Utf8, false),
- Field::new("num", DataType::Int32, false),
- ]);
- arrow_fields.push(Field::new(
- "my_list",
- DataType::List(Box::new(Field::new("element", arrow_struct, true))),
- true,
- ));
- }
-
- // // List<OneTuple<String>> (nullable list, non-null elements)
- // optional group my_list (LIST) {
- // repeated group array {
- // required binary str (UTF8);
- // };
- // }
- // Special case: group is named array
- {
- let arrow_struct =
- DataType::Struct(vec![Field::new("str", DataType::Utf8, false)]);
- arrow_fields.push(Field::new(
- "my_list",
- DataType::List(Box::new(Field::new("array", arrow_struct, true))),
- true,
- ));
- }
-
- // // List<OneTuple<String>> (nullable list, non-null elements)
- // optional group my_list (LIST) {
- // repeated group my_list_tuple {
- // required binary str (UTF8);
- // };
- // }
- // Special case: group named ends in _tuple
- {
- let arrow_struct =
- DataType::Struct(vec![Field::new("str", DataType::Utf8, false)]);
- arrow_fields.push(Field::new(
- "my_list",
- DataType::List(Box::new(Field::new("my_list_tuple", arrow_struct, true))),
- true,
- ));
- }
-
- // One-level encoding: Only allows required lists with required cells
- // repeated value_type name
- {
- arrow_fields.push(Field::new(
- "name",
- DataType::List(Box::new(Field::new("name", DataType::Int32, true))),
- true,
- ));
- }
-
- let parquet_group_type = parse_message_type(message_type).unwrap();
-
- let parquet_schema = SchemaDescriptor::new(Arc::new(parquet_group_type));
- let converted_arrow_schema =
- parquet_to_arrow_schema(&parquet_schema, &None).unwrap();
- let converted_fields = converted_arrow_schema.fields();
-
- assert_eq!(arrow_fields.len(), converted_fields.len());
- for i in 0..arrow_fields.len() {
- assert_eq!(arrow_fields[i], converted_fields[i]);
- }
- }
-
- #[test]
- fn test_parquet_list_nullable() {
- let mut arrow_fields = Vec::new();
-
- let message_type = "
- message test_schema {
- REQUIRED GROUP my_list1 (LIST) {
- REPEATED GROUP list {
- OPTIONAL BINARY element (UTF8);
- }
- }
- OPTIONAL GROUP my_list2 (LIST) {
- REPEATED GROUP list {
- REQUIRED BINARY element (UTF8);
- }
- }
- REQUIRED GROUP my_list3 (LIST) {
- REPEATED GROUP list {
- REQUIRED BINARY element (UTF8);
- }
- }
- }
- ";
-
- // // List<String> (list non-null, elements nullable)
- // required group my_list1 (LIST) {
- // repeated group list {
- // optional binary element (UTF8);
- // }
- // }
- {
- arrow_fields.push(Field::new(
- "my_list1",
- DataType::List(Box::new(Field::new("element", DataType::Utf8, true))),
- false,
- ));
- }
-
- // // List<String> (list nullable, elements non-null)
- // optional group my_list2 (LIST) {
- // repeated group list {
- // required binary element (UTF8);
- // }
- // }
- {
- arrow_fields.push(Field::new(
- "my_list2",
- DataType::List(Box::new(Field::new("element", DataType::Utf8, false))),
- true,
- ));
- }
-
- // // List<String> (list non-null, elements non-null)
- // repeated group my_list3 (LIST) {
- // repeated group list {
- // required binary element (UTF8);
- // }
- // }
- {
- arrow_fields.push(Field::new(
- "my_list3",
- DataType::List(Box::new(Field::new("element", DataType::Utf8, false))),
- false,
- ));
- }
-
- let parquet_group_type = parse_message_type(message_type).unwrap();
-
- let parquet_schema = SchemaDescriptor::new(Arc::new(parquet_group_type));
- let converted_arrow_schema =
- parquet_to_arrow_schema(&parquet_schema, &None).unwrap();
- let converted_fields = converted_arrow_schema.fields();
-
- assert_eq!(arrow_fields.len(), converted_fields.len());
- for i in 0..arrow_fields.len() {
- assert_eq!(arrow_fields[i], converted_fields[i]);
- }
- }
-
- #[test]
- fn test_nested_schema() {
- let mut arrow_fields = Vec::new();
- {
- let group1_fields = vec![
- Field::new("leaf1", DataType::Boolean, false),
- Field::new("leaf2", DataType::Int32, false),
- ];
- let group1_struct =
- Field::new("group1", DataType::Struct(group1_fields), false);
- arrow_fields.push(group1_struct);
-
- let leaf3_field = Field::new("leaf3", DataType::Int64, false);
- arrow_fields.push(leaf3_field);
- }
-
- let message_type = "
- message test_schema {
- REQUIRED GROUP group1 {
- REQUIRED BOOLEAN leaf1;
- REQUIRED INT32 leaf2;
- }
- REQUIRED INT64 leaf3;
- }
- ";
- let parquet_group_type = parse_message_type(message_type).unwrap();
-
- let parquet_schema = SchemaDescriptor::new(Arc::new(parquet_group_type));
- let converted_arrow_schema =
- parquet_to_arrow_schema(&parquet_schema, &None).unwrap();
- let converted_fields = converted_arrow_schema.fields();
-
- assert_eq!(arrow_fields.len(), converted_fields.len());
- for i in 0..arrow_fields.len() {
- assert_eq!(arrow_fields[i], converted_fields[i]);
- }
- }
-
- #[test]
- fn test_nested_schema_partial() {
- let mut arrow_fields = Vec::new();
- {
- let group1_fields = vec![Field::new("leaf1", DataType::Int64, false)];
- let group1 = Field::new("group1", DataType::Struct(group1_fields), false);
- arrow_fields.push(group1);
-
- let group2_fields = vec![Field::new("leaf4", DataType::Int64, false)];
- let group2 = Field::new("group2", DataType::Struct(group2_fields), false);
- arrow_fields.push(group2);
-
- arrow_fields.push(Field::new("leaf5", DataType::Int64, false));
- }
-
- let message_type = "
- message test_schema {
- REQUIRED GROUP group1 {
- REQUIRED INT64 leaf1;
- REQUIRED INT64 leaf2;
- }
- REQUIRED GROUP group2 {
- REQUIRED INT64 leaf3;
- REQUIRED INT64 leaf4;
- }
- REQUIRED INT64 leaf5;
- }
- ";
- let parquet_group_type = parse_message_type(message_type).unwrap();
-
- // Expected partial arrow schema (columns 0, 3, 4):
- // required group group1 {
- // required int64 leaf1;
- // }
- // required group group2 {
- // required int64 leaf4;
- // }
- // required int64 leaf5;
-
- let parquet_schema = SchemaDescriptor::new(Arc::new(parquet_group_type));
- let converted_arrow_schema =
- parquet_to_arrow_schema_by_columns(&parquet_schema, vec![0, 3, 4], &None)
- .unwrap();
- let converted_fields = converted_arrow_schema.fields();
-
- assert_eq!(arrow_fields.len(), converted_fields.len());
- for i in 0..arrow_fields.len() {
- assert_eq!(arrow_fields[i], converted_fields[i]);
- }
- }
-
- #[test]
- fn test_nested_schema_partial_ordering() {
- let mut arrow_fields = Vec::new();
- {
- let group2_fields = vec![Field::new("leaf4", DataType::Int64, false)];
- let group2 = Field::new("group2", DataType::Struct(group2_fields), false);
- arrow_fields.push(group2);
-
- arrow_fields.push(Field::new("leaf5", DataType::Int64, false));
-
- let group1_fields = vec![Field::new("leaf1", DataType::Int64, false)];
- let group1 = Field::new("group1", DataType::Struct(group1_fields), false);
- arrow_fields.push(group1);
- }
-
- let message_type = "
- message test_schema {
- REQUIRED GROUP group1 {
- REQUIRED INT64 leaf1;
- REQUIRED INT64 leaf2;
- }
- REQUIRED GROUP group2 {
- REQUIRED INT64 leaf3;
- REQUIRED INT64 leaf4;
- }
- REQUIRED INT64 leaf5;
- }
- ";
- let parquet_group_type = parse_message_type(message_type).unwrap();
-
- // Expected partial arrow schema (columns 3, 4, 0):
- // required group group1 {
- // required int64 leaf1;
- // }
- // required group group2 {
- // required int64 leaf4;
- // }
- // required int64 leaf5;
-
- let parquet_schema = SchemaDescriptor::new(Arc::new(parquet_group_type));
- let converted_arrow_schema =
- parquet_to_arrow_schema_by_columns(&parquet_schema, vec![3, 4, 0], &None)
- .unwrap();
- let converted_fields = converted_arrow_schema.fields();
-
- assert_eq!(arrow_fields.len(), converted_fields.len());
- for i in 0..arrow_fields.len() {
- assert_eq!(arrow_fields[i], converted_fields[i]);
- }
- }
-
- #[test]
- fn test_repeated_nested_schema() {
- let mut arrow_fields = Vec::new();
- {
- arrow_fields.push(Field::new("leaf1", DataType::Int32, true));
-
- let inner_group_list = Field::new(
- "innerGroup",
- DataType::List(Box::new(Field::new(
- "innerGroup",
- DataType::Struct(vec![Field::new("leaf3", DataType::Int32, true)]),
- true,
- ))),
- true,
- );
-
- let outer_group_list = Field::new(
- "outerGroup",
- DataType::List(Box::new(Field::new(
- "outerGroup",
- DataType::Struct(vec![
- Field::new("leaf2", DataType::Int32, true),
- inner_group_list,
- ]),
- true,
- ))),
- true,
- );
- arrow_fields.push(outer_group_list);
- }
-
- let message_type = "
- message test_schema {
- OPTIONAL INT32 leaf1;
- REPEATED GROUP outerGroup {
- OPTIONAL INT32 leaf2;
- REPEATED GROUP innerGroup {
- OPTIONAL INT32 leaf3;
- }
- }
- }
- ";
- let parquet_group_type = parse_message_type(message_type).unwrap();
-
- let parquet_schema = SchemaDescriptor::new(Arc::new(parquet_group_type));
- let converted_arrow_schema =
- parquet_to_arrow_schema(&parquet_schema, &None).unwrap();
- let converted_fields = converted_arrow_schema.fields();
-
- assert_eq!(arrow_fields.len(), converted_fields.len());
- for i in 0..arrow_fields.len() {
- assert_eq!(arrow_fields[i], converted_fields[i]);
- }
- }
-
- #[test]
- fn test_column_desc_to_field() {
- let message_type = "
- message test_schema {
- REQUIRED BOOLEAN boolean;
- REQUIRED INT32 int8 (INT_8);
- REQUIRED INT32 uint8 (INTEGER(8,false));
- REQUIRED INT32 int16 (INT_16);
- REQUIRED INT32 uint16 (INTEGER(16,false));
- REQUIRED INT32 int32;
- REQUIRED INT64 int64;
- OPTIONAL DOUBLE double;
- OPTIONAL FLOAT float;
- OPTIONAL BINARY string (UTF8);
- REPEATED BOOLEAN bools;
- OPTIONAL INT32 date (DATE);
- OPTIONAL INT32 time_milli (TIME_MILLIS);
- OPTIONAL INT64 time_micro (TIME_MICROS);
- OPTIONAL INT64 time_nano (TIME(NANOS,false));
- OPTIONAL INT64 ts_milli (TIMESTAMP_MILLIS);
- REQUIRED INT64 ts_micro (TIMESTAMP_MICROS);
- REQUIRED INT64 ts_nano (TIMESTAMP(NANOS,true));
- }
- ";
- let parquet_group_type = parse_message_type(message_type).unwrap();
-
- let parquet_schema = SchemaDescriptor::new(Arc::new(parquet_group_type));
- let converted_arrow_fields = parquet_schema
- .columns()
- .iter()
- .map(|c| parquet_to_arrow_field(c).unwrap())
- .collect::<Vec<Field>>();
-
- let arrow_fields = vec![
- Field::new("boolean", DataType::Boolean, false),
- Field::new("int8", DataType::Int8, false),
- Field::new("uint8", DataType::UInt8, false),
- Field::new("int16", DataType::Int16, false),
- Field::new("uint16", DataType::UInt16, false),
- Field::new("int32", DataType::Int32, false),
- Field::new("int64", DataType::Int64, false),
- Field::new("double", DataType::Float64, true),
- Field::new("float", DataType::Float32, true),
- Field::new("string", DataType::Utf8, true),
- Field::new(
- "bools",
- DataType::List(Box::new(Field::new("bools", DataType::Boolean, true))),
- true,
- ),
- Field::new("date", DataType::Date32, true),
- Field::new("time_milli", DataType::Time32(TimeUnit::Millisecond), true),
- Field::new("time_micro", DataType::Time64(TimeUnit::Microsecond), true),
- Field::new("time_nano", DataType::Time64(TimeUnit::Nanosecond), true),
- Field::new(
- "ts_milli",
- DataType::Timestamp(TimeUnit::Millisecond, None),
- true,
- ),
- Field::new(
- "ts_micro",
- DataType::Timestamp(TimeUnit::Microsecond, None),
- false,
- ),
- Field::new(
- "ts_nano",
- DataType::Timestamp(TimeUnit::Nanosecond, Some("UTC".to_string())),
- false,
- ),
- ];
-
- assert_eq!(arrow_fields, converted_arrow_fields);
- }
-
- #[test]
- fn test_field_to_column_desc() {
- let message_type = "
- message arrow_schema {
- REQUIRED BOOLEAN boolean;
- REQUIRED INT32 int8 (INT_8);
- REQUIRED INT32 int16 (INTEGER(16,true));
- REQUIRED INT32 int32;
- REQUIRED INT64 int64;
- OPTIONAL DOUBLE double;
- OPTIONAL FLOAT float;
- OPTIONAL BINARY string (STRING);
- OPTIONAL GROUP bools (LIST) {
- REPEATED GROUP list {
- OPTIONAL BOOLEAN element;
- }
- }
- REQUIRED GROUP bools_non_null (LIST) {
- REPEATED GROUP list {
- REQUIRED BOOLEAN element;
- }
- }
- OPTIONAL INT32 date (DATE);
- OPTIONAL INT32 time_milli (TIME(MILLIS,false));
- OPTIONAL INT64 time_micro (TIME_MICROS);
- OPTIONAL INT64 ts_milli (TIMESTAMP_MILLIS);
- REQUIRED INT64 ts_micro (TIMESTAMP(MICROS,false));
- REQUIRED GROUP struct {
- REQUIRED BOOLEAN bools;
- REQUIRED INT32 uint32 (INTEGER(32,false));
- REQUIRED GROUP int32 (LIST) {
- REPEATED GROUP list {
- OPTIONAL INT32 element;
- }
- }
- }
- REQUIRED BINARY dictionary_strings (STRING);
- }
- ";
- let parquet_group_type = parse_message_type(message_type).unwrap();
-
- let parquet_schema = SchemaDescriptor::new(Arc::new(parquet_group_type));
-
- let arrow_fields = vec![
- Field::new("boolean", DataType::Boolean, false),
- Field::new("int8", DataType::Int8, false),
- Field::new("int16", DataType::Int16, false),
- Field::new("int32", DataType::Int32, false),
- Field::new("int64", DataType::Int64, false),
- Field::new("double", DataType::Float64, true),
- Field::new("float", DataType::Float32, true),
- Field::new("string", DataType::Utf8, true),
- Field::new(
- "bools",
- DataType::List(Box::new(Field::new("element", DataType::Boolean, true))),
- true,
- ),
- Field::new(
- "bools_non_null",
- DataType::List(Box::new(Field::new("element", DataType::Boolean, false))),
- false,
- ),
- Field::new("date", DataType::Date32, true),
- Field::new("time_milli", DataType::Time32(TimeUnit::Millisecond), true),
- Field::new("time_micro", DataType::Time64(TimeUnit::Microsecond), true),
- Field::new(
- "ts_milli",
- DataType::Timestamp(TimeUnit::Millisecond, None),
- true,
- ),
- Field::new(
- "ts_micro",
- DataType::Timestamp(TimeUnit::Microsecond, None),
- false,
- ),
- Field::new(
- "struct",
- DataType::Struct(vec![
- Field::new("bools", DataType::Boolean, false),
- Field::new("uint32", DataType::UInt32, false),
- Field::new(
- "int32",
- DataType::List(Box::new(Field::new(
- "element",
- DataType::Int32,
- true,
- ))),
- false,
- ),
- ]),
- false,
- ),
- Field::new(
- "dictionary_strings",
- DataType::Dictionary(Box::new(DataType::Int32), Box::new(DataType::Utf8)),
- false,
- ),
- ];
- let arrow_schema = Schema::new(arrow_fields);
- let converted_arrow_schema = arrow_to_parquet_schema(&arrow_schema).unwrap();
-
- assert_eq!(
- parquet_schema.columns().len(),
- converted_arrow_schema.columns().len()
- );
- parquet_schema
- .columns()
- .iter()
- .zip(converted_arrow_schema.columns())
- .for_each(|(a, b)| {
- // Only check logical type if it's set on the Parquet side.
- // This is because the Arrow conversion always sets logical type,
- // even if there wasn't originally one.
- // This is not an issue, but is an inconvenience for this test.
- match a.logical_type() {
- Some(_) => {
- assert_eq!(a, b)
- }
- None => {
- assert_eq!(a.name(), b.name());
- assert_eq!(a.physical_type(), b.physical_type());
- assert_eq!(a.converted_type(), b.converted_type());
- }
- };
- });
- }
-
- #[test]
- #[should_panic(expected = "Parquet does not support writing empty structs")]
- fn test_empty_struct_field() {
- let arrow_fields = vec![Field::new("struct", DataType::Struct(vec![]), false)];
- let arrow_schema = Schema::new(arrow_fields);
- let converted_arrow_schema = arrow_to_parquet_schema(&arrow_schema);
-
- assert!(converted_arrow_schema.is_err());
- converted_arrow_schema.unwrap();
- }
-
- #[test]
- fn test_metadata() {
- let message_type = "
- message test_schema {
- OPTIONAL BINARY string (STRING);
- }
- ";
- let parquet_group_type = parse_message_type(message_type).unwrap();
-
- let mut key_value_metadata: Vec<KeyValue> = Vec::new();
- key_value_metadata.push(KeyValue::new("foo".to_owned(), Some("bar".to_owned())));
- key_value_metadata.push(KeyValue::new("baz".to_owned(), None));
-
- let mut expected_metadata: HashMap<String, String> = HashMap::new();
- expected_metadata.insert("foo".to_owned(), "bar".to_owned());
-
- let parquet_schema = SchemaDescriptor::new(Arc::new(parquet_group_type));
- let converted_arrow_schema =
- parquet_to_arrow_schema(&parquet_schema, &Some(key_value_metadata)).unwrap();
-
- assert_eq!(converted_arrow_schema.metadata(), &expected_metadata);
- }
-
- #[test]
- fn test_arrow_schema_roundtrip() -> Result<()> {
- // This tests the roundtrip of an Arrow schema
- // Fields that are commented out fail roundtrip tests or are unsupported by the writer
- let metadata: HashMap<String, String> =
- [("Key".to_string(), "Value".to_string())]
- .iter()
- .cloned()
- .collect();
-
- let schema = Schema::new_with_metadata(
- vec![
- Field::new("c1", DataType::Utf8, false),
- Field::new("c2", DataType::Binary, false),
- Field::new("c3", DataType::FixedSizeBinary(3), false),
- Field::new("c4", DataType::Boolean, false),
- Field::new("c5", DataType::Date32, false),
- Field::new("c6", DataType::Date64, false),
- Field::new("c7", DataType::Time32(TimeUnit::Second), false),
- Field::new("c8", DataType::Time32(TimeUnit::Millisecond), false),
- Field::new("c13", DataType::Time64(TimeUnit::Microsecond), false),
- Field::new("c14", DataType::Time64(TimeUnit::Nanosecond), false),
- Field::new("c15", DataType::Timestamp(TimeUnit::Second, None), false),
- Field::new(
- "c16",
- DataType::Timestamp(TimeUnit::Millisecond, Some("UTC".to_string())),
- false,
- ),
- Field::new(
- "c17",
- DataType::Timestamp(
- TimeUnit::Microsecond,
- Some("Africa/Johannesburg".to_string()),
- ),
- false,
- ),
- Field::new(
- "c18",
- DataType::Timestamp(TimeUnit::Nanosecond, None),
- false,
- ),
- Field::new("c19", DataType::Interval(IntervalUnit::DayTime), false),
- Field::new("c20", DataType::Interval(IntervalUnit::YearMonth), false),
- Field::new(
- "c21",
- DataType::List(Box::new(Field::new("list", DataType::Boolean, true))),
- false,
- ),
- // Field::new(
- // "c22",
- // DataType::FixedSizeList(Box::new(DataType::Boolean), 5),
- // false,
- // ),
- // Field::new(
- // "c23",
- // DataType::List(Box::new(DataType::LargeList(Box::new(
- // DataType::Struct(vec![
- // Field::new("a", DataType::Int16, true),
- // Field::new("b", DataType::Float64, false),
- // ]),
- // )))),
- // true,
- // ),
- Field::new(
- "c24",
- DataType::Struct(vec![
- Field::new("a", DataType::Utf8, false),
- Field::new("b", DataType::UInt16, false),
- ]),
- false,
- ),
- Field::new("c25", DataType::Interval(IntervalUnit::YearMonth), true),
- Field::new("c26", DataType::Interval(IntervalUnit::DayTime), true),
- // Field::new("c27", DataType::Duration(TimeUnit::Second), false),
- // Field::new("c28", DataType::Duration(TimeUnit::Millisecond), false),
- // Field::new("c29", DataType::Duration(TimeUnit::Microsecond), false),
- // Field::new("c30", DataType::Duration(TimeUnit::Nanosecond), false),
- Field::new_dict(
- "c31",
- DataType::Dictionary(
- Box::new(DataType::Int32),
- Box::new(DataType::Utf8),
- ),
- true,
- 123,
- true,
- ),
- Field::new("c32", DataType::LargeBinary, true),
- Field::new("c33", DataType::LargeUtf8, true),
- // Field::new(
- // "c34",
- // DataType::LargeList(Box::new(DataType::List(Box::new(
- // DataType::Struct(vec![
- // Field::new("a", DataType::Int16, true),
- // Field::new("b", DataType::Float64, true),
- // ]),
- // )))),
- // true,
- // ),
- Field::new("c35", DataType::Null, true),
- Field::new("c36", DataType::Decimal(2, 1), false),
- Field::new("c37", DataType::Decimal(50, 20), false),
- Field::new("c38", DataType::Decimal(18, 12), true),
- ],
- metadata,
- );
-
- // write to an empty parquet file so that schema is serialized
- let file = get_temp_file("test_arrow_schema_roundtrip.parquet", &[]);
- let mut writer = ArrowWriter::try_new(
- file.try_clone().unwrap(),
- Arc::new(schema.clone()),
- None,
- )?;
- writer.close()?;
-
- // read file back
- let parquet_reader = SerializedFileReader::try_from(file)?;
- let mut arrow_reader = ParquetFileArrowReader::new(Arc::new(parquet_reader));
- let read_schema = arrow_reader.get_schema()?;
- assert_eq!(schema, read_schema);
-
- // read all fields by columns
- let partial_read_schema =
- arrow_reader.get_schema_by_columns(0..(schema.fields().len()), false)?;
- assert_eq!(schema, partial_read_schema);
-
- Ok(())
- }
-
- #[test]
- #[ignore = "Roundtrip of lists currently fails because we don't check their types correctly in the Arrow schema"]
- fn test_arrow_schema_roundtrip_lists() -> Result<()> {
- let metadata: HashMap<String, String> =
- [("Key".to_string(), "Value".to_string())]
- .iter()
- .cloned()
- .collect();
-
- let schema = Schema::new_with_metadata(
- vec![
- Field::new(
- "c21",
- DataType::List(Box::new(Field::new(
- "array",
- DataType::Boolean,
- true,
- ))),
- false,
- ),
- Field::new(
- "c22",
- DataType::FixedSizeList(
- Box::new(Field::new("items", DataType::Boolean, false)),
- 5,
- ),
- false,
- ),
- Field::new(
- "c23",
- DataType::List(Box::new(Field::new(
- "items",
- DataType::LargeList(Box::new(Field::new(
- "items",
- DataType::Struct(vec![
- Field::new("a", DataType::Int16, true),
- Field::new("b", DataType::Float64, false),
- ]),
- true,
- ))),
- true,
- ))),
- true,
- ),
- ],
- metadata,
- );
-
- // write to an empty parquet file so that schema is serialized
- let file = get_temp_file("test_arrow_schema_roundtrip_lists.parquet", &[]);
- let mut writer = ArrowWriter::try_new(
- file.try_clone().unwrap(),
- Arc::new(schema.clone()),
- None,
- )?;
- writer.close()?;
-
- // read file back
- let parquet_reader = SerializedFileReader::try_from(file)?;
- let mut arrow_reader = ParquetFileArrowReader::new(Arc::new(parquet_reader));
- let read_schema = arrow_reader.get_schema()?;
- assert_eq!(schema, read_schema);
-
- // read all fields by columns
- let partial_read_schema =
- arrow_reader.get_schema_by_columns(0..(schema.fields().len()), false)?;
- assert_eq!(schema, partial_read_schema);
-
- Ok(())
- }
-}
diff --git a/parquet/src/basic.rs b/parquet/src/basic.rs
deleted file mode 100644
index 631257e..0000000
--- a/parquet/src/basic.rs
+++ /dev/null
@@ -1,1969 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-//! Contains Rust mappings for Thrift definition.
-//! Refer to `parquet.thrift` file to see raw definitions.
-
-use std::{convert, fmt, result, str};
-
-use parquet_format as parquet;
-
-use crate::errors::ParquetError;
-
-// Re-export parquet_format types used in this module
-pub use parquet_format::{
- BsonType, DateType, DecimalType, EnumType, IntType, JsonType, ListType, MapType,
- NullType, StringType, TimeType, TimeUnit, TimestampType, UUIDType,
-};
-
-// ----------------------------------------------------------------------
-// Types from the Thrift definition
-
-// ----------------------------------------------------------------------
-// Mirrors `parquet::Type`
-
-/// Types supported by Parquet.
-/// These physical types are intended to be used in combination with the encodings to
-/// control the on disk storage format.
-/// For example INT16 is not included as a type since a good encoding of INT32
-/// would handle this.
-#[derive(Debug, Clone, Copy, PartialEq)]
-pub enum Type {
- BOOLEAN,
- INT32,
- INT64,
- INT96,
- FLOAT,
- DOUBLE,
- BYTE_ARRAY,
- FIXED_LEN_BYTE_ARRAY,
-}
-
-// ----------------------------------------------------------------------
-// Mirrors `parquet::ConvertedType`
-
-/// Common types (converted types) used by frameworks when using Parquet.
-/// This helps map between types in those frameworks to the base types in Parquet.
-/// This is only metadata and not needed to read or write the data.
-///
-/// This struct was renamed from `LogicalType` in version 4.0.0.
-/// If targeting Parquet format 2.4.0 or above, please use [LogicalType] instead.
-#[derive(Debug, Clone, Copy, PartialEq)]
-pub enum ConvertedType {
- NONE,
- /// A BYTE_ARRAY actually contains UTF8 encoded chars.
- UTF8,
-
- /// A map is converted as an optional field containing a repeated key/value pair.
- MAP,
-
- /// A key/value pair is converted into a group of two fields.
- MAP_KEY_VALUE,
-
- /// A list is converted into an optional field containing a repeated field for its
- /// values.
- LIST,
-
- /// An enum is converted into a binary field
- ENUM,
-
- /// A decimal value.
- /// This may be used to annotate binary or fixed primitive types. The
- /// underlying byte array stores the unscaled value encoded as two's
- /// complement using big-endian byte order (the most significant byte is the
- /// zeroth element).
- ///
- /// This must be accompanied by a (maximum) precision and a scale in the
- /// SchemaElement. The precision specifies the number of digits in the decimal
- /// and the scale stores the location of the decimal point. For example 1.23
- /// would have precision 3 (3 total digits) and scale 2 (the decimal point is
- /// 2 digits over).
- DECIMAL,
-
- /// A date stored as days since Unix epoch, encoded as the INT32 physical type.
- DATE,
-
- /// The total number of milliseconds since midnight. The value is stored as an INT32
- /// physical type.
- TIME_MILLIS,
-
- /// The total number of microseconds since midnight. The value is stored as an INT64
- /// physical type.
- TIME_MICROS,
-
- /// Date and time recorded as milliseconds since the Unix epoch.
- /// Recorded as a physical type of INT64.
- TIMESTAMP_MILLIS,
-
- /// Date and time recorded as microseconds since the Unix epoch.
- /// The value is stored as an INT64 physical type.
- TIMESTAMP_MICROS,
-
- /// An unsigned 8 bit integer value stored as INT32 physical type.
- UINT_8,
-
- /// An unsigned 16 bit integer value stored as INT32 physical type.
- UINT_16,
-
- /// An unsigned 32 bit integer value stored as INT32 physical type.
- UINT_32,
-
- /// An unsigned 64 bit integer value stored as INT64 physical type.
- UINT_64,
-
- /// A signed 8 bit integer value stored as INT32 physical type.
- INT_8,
-
- /// A signed 16 bit integer value stored as INT32 physical type.
- INT_16,
-
- /// A signed 32 bit integer value stored as INT32 physical type.
- INT_32,
-
- /// A signed 64 bit integer value stored as INT64 physical type.
- INT_64,
-
- /// A JSON document embedded within a single UTF8 column.
- JSON,
-
- /// A BSON document embedded within a single BINARY column.
- BSON,
-
- /// An interval of time.
- ///
- /// This type annotates data stored as a FIXED_LEN_BYTE_ARRAY of length 12.
- /// This data is composed of three separate little endian unsigned integers.
- /// Each stores a component of a duration of time. The first integer identifies
- /// the number of months associated with the duration, the second identifies
- /// the number of days associated with the duration and the third identifies
- /// the number of milliseconds associated with the provided duration.
- /// This duration of time is independent of any particular timezone or date.
- INTERVAL,
-}
-
-// ----------------------------------------------------------------------
-// Mirrors `parquet::LogicalType`
-
-/// Logical types used by version 2.4.0+ of the Parquet format.
-///
-/// This is an *entirely new* struct as of version
-/// 4.0.0. The struct previously named `LogicalType` was renamed to
-/// [`ConvertedType`]. Please see the README.md for more details.
-#[derive(Debug, Clone, PartialEq)]
-pub enum LogicalType {
- STRING(StringType),
- MAP(MapType),
- LIST(ListType),
- ENUM(EnumType),
- DECIMAL(DecimalType),
- DATE(DateType),
- TIME(TimeType),
- TIMESTAMP(TimestampType),
- INTEGER(IntType),
- UNKNOWN(NullType),
- JSON(JsonType),
- BSON(BsonType),
- UUID(UUIDType),
-}
-
-// ----------------------------------------------------------------------
-// Mirrors `parquet::FieldRepetitionType`
-
-/// Representation of field types in schema.
-#[derive(Debug, Clone, Copy, PartialEq)]
-pub enum Repetition {
- /// Field is required (can not be null) and each record has exactly 1 value.
- REQUIRED,
- /// Field is optional (can be null) and each record has 0 or 1 values.
- OPTIONAL,
- /// Field is repeated and can contain 0 or more values.
- REPEATED,
-}
-
-// ----------------------------------------------------------------------
-// Mirrors `parquet::Encoding`
-
-/// Encodings supported by Parquet.
-/// Not all encodings are valid for all types. These enums are also used to specify the
-/// encoding of definition and repetition levels.
-#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
-pub enum Encoding {
- /// Default byte encoding.
- /// - BOOLEAN - 1 bit per value, 0 is false; 1 is true.
- /// - INT32 - 4 bytes per value, stored as little-endian.
- /// - INT64 - 8 bytes per value, stored as little-endian.
- /// - FLOAT - 4 bytes per value, stored as little-endian.
- /// - DOUBLE - 8 bytes per value, stored as little-endian.
- /// - BYTE_ARRAY - 4 byte length stored as little endian, followed by bytes.
- /// - FIXED_LEN_BYTE_ARRAY - just the bytes are stored.
- PLAIN,
-
- /// **Deprecated** dictionary encoding.
- ///
- /// The values in the dictionary are encoded using PLAIN encoding.
- /// Since it is deprecated, RLE_DICTIONARY encoding is used for a data page, and
- /// PLAIN encoding is used for dictionary page.
- PLAIN_DICTIONARY,
-
- /// Group packed run length encoding.
- ///
- /// Usable for definition/repetition levels encoding and boolean values.
- RLE,
-
- /// Bit packed encoding.
- ///
- /// This can only be used if the data has a known max width.
- /// Usable for definition/repetition levels encoding.
- BIT_PACKED,
-
- /// Delta encoding for integers, either INT32 or INT64.
- ///
- /// Works best on sorted data.
- DELTA_BINARY_PACKED,
-
- /// Encoding for byte arrays to separate the length values and the data.
- ///
- /// The lengths are encoded using DELTA_BINARY_PACKED encoding.
- DELTA_LENGTH_BYTE_ARRAY,
-
- /// Incremental encoding for byte arrays.
- ///
- /// Prefix lengths are encoded using DELTA_BINARY_PACKED encoding.
- /// Suffixes are stored using DELTA_LENGTH_BYTE_ARRAY encoding.
- DELTA_BYTE_ARRAY,
-
- /// Dictionary encoding.
- ///
- /// The ids are encoded using the RLE encoding.
- RLE_DICTIONARY,
-}
-
-// ----------------------------------------------------------------------
-// Mirrors `parquet::CompressionCodec`
-
-/// Supported compression algorithms.
-#[derive(Debug, Clone, Copy, PartialEq)]
-pub enum Compression {
- UNCOMPRESSED,
- SNAPPY,
- GZIP,
- LZO,
- BROTLI,
- LZ4,
- ZSTD,
-}
-
-// ----------------------------------------------------------------------
-// Mirrors `parquet::PageType`
-
-/// Available data pages for Parquet file format.
-/// Note that some of the page types may not be supported.
-#[derive(Debug, Clone, Copy, PartialEq)]
-pub enum PageType {
- DATA_PAGE,
- INDEX_PAGE,
- DICTIONARY_PAGE,
- DATA_PAGE_V2,
-}
-
-// ----------------------------------------------------------------------
-// Mirrors `parquet::ColumnOrder`
-
-/// Sort order for page and column statistics.
-///
-/// Types are associated with sort orders and column stats are aggregated using a sort
-/// order, and a sort order should be considered when comparing values with statistics
-/// min/max.
-///
-/// See reference in
-/// <https://github.com/apache/parquet-cpp/blob/master/src/parquet/types.h>
-#[derive(Debug, Clone, Copy, PartialEq)]
-pub enum SortOrder {
- /// Signed (either value or legacy byte-wise) comparison.
- SIGNED,
- /// Unsigned (depending on physical type either value or byte-wise) comparison.
- UNSIGNED,
- /// Comparison is undefined.
- UNDEFINED,
-}
-
-/// Column order that specifies what method was used to aggregate min/max values for
-/// statistics.
-///
-/// If column order is undefined, then it is the legacy behaviour and all values should
-/// be compared as signed values/bytes.
-#[derive(Debug, Clone, Copy, PartialEq)]
-pub enum ColumnOrder {
- /// Column uses the order defined by its logical or physical type
- /// (if there is no logical type), parquet-format 2.4.0+.
- TYPE_DEFINED_ORDER(SortOrder),
- /// Undefined column order, means legacy behaviour before parquet-format 2.4.0.
- /// Sort order is always SIGNED.
- UNDEFINED,
-}
-
-impl ColumnOrder {
- /// Returns sort order for a physical/logical type.
- pub fn get_sort_order(
- logical_type: Option<LogicalType>,
- converted_type: ConvertedType,
- physical_type: Type,
- ) -> SortOrder {
- // TODO: Should this take converted and logical type, for compatibility?
- match logical_type {
- Some(logical) => match logical {
- LogicalType::STRING(_)
- | LogicalType::ENUM(_)
- | LogicalType::JSON(_)
- | LogicalType::BSON(_) => SortOrder::UNSIGNED,
- LogicalType::INTEGER(t) => match t.is_signed {
- true => SortOrder::SIGNED,
- false => SortOrder::UNSIGNED,
- },
- LogicalType::MAP(_) | LogicalType::LIST(_) => SortOrder::UNDEFINED,
- LogicalType::DECIMAL(_) => SortOrder::SIGNED,
- LogicalType::DATE(_) => SortOrder::SIGNED,
- LogicalType::TIME(_) => SortOrder::SIGNED,
- LogicalType::TIMESTAMP(_) => SortOrder::SIGNED,
- LogicalType::UNKNOWN(_) => SortOrder::UNDEFINED,
- LogicalType::UUID(_) => SortOrder::UNSIGNED,
- },
- // Fall back to converted type
- None => Self::get_converted_sort_order(converted_type, physical_type),
- }
- }
-
- fn get_converted_sort_order(
- converted_type: ConvertedType,
- physical_type: Type,
- ) -> SortOrder {
- match converted_type {
- // Unsigned byte-wise comparison.
- ConvertedType::UTF8
- | ConvertedType::JSON
- | ConvertedType::BSON
- | ConvertedType::ENUM => SortOrder::UNSIGNED,
-
- ConvertedType::INT_8
- | ConvertedType::INT_16
- | ConvertedType::INT_32
- | ConvertedType::INT_64 => SortOrder::SIGNED,
-
- ConvertedType::UINT_8
- | ConvertedType::UINT_16
- | ConvertedType::UINT_32
- | ConvertedType::UINT_64 => SortOrder::UNSIGNED,
-
- // Signed comparison of the represented value.
- ConvertedType::DECIMAL => SortOrder::SIGNED,
-
- ConvertedType::DATE => SortOrder::SIGNED,
-
- ConvertedType::TIME_MILLIS
- | ConvertedType::TIME_MICROS
- | ConvertedType::TIMESTAMP_MILLIS
- | ConvertedType::TIMESTAMP_MICROS => SortOrder::SIGNED,
-
- ConvertedType::INTERVAL => SortOrder::UNDEFINED,
-
- ConvertedType::LIST | ConvertedType::MAP | ConvertedType::MAP_KEY_VALUE => {
- SortOrder::UNDEFINED
- }
-
- // Fall back to physical type.
- ConvertedType::NONE => Self::get_default_sort_order(physical_type),
- }
- }
-
- /// Returns default sort order based on physical type.
- fn get_default_sort_order(physical_type: Type) -> SortOrder {
- match physical_type {
- // Order: false, true
- Type::BOOLEAN => SortOrder::UNSIGNED,
- Type::INT32 | Type::INT64 => SortOrder::SIGNED,
- Type::INT96 => SortOrder::UNDEFINED,
- // Notes to remember when comparing float/double values:
- // If the min is a NaN, it should be ignored.
- // If the max is a NaN, it should be ignored.
- // If the min is +0, the row group may contain -0 values as well.
- // If the max is -0, the row group may contain +0 values as well.
- // When looking for NaN values, min and max should be ignored.
- Type::FLOAT | Type::DOUBLE => SortOrder::SIGNED,
- // Unsigned byte-wise comparison
- Type::BYTE_ARRAY | Type::FIXED_LEN_BYTE_ARRAY => SortOrder::UNSIGNED,
- }
- }
-
- /// Returns sort order associated with this column order.
- pub fn sort_order(&self) -> SortOrder {
- match *self {
- ColumnOrder::TYPE_DEFINED_ORDER(order) => order,
- ColumnOrder::UNDEFINED => SortOrder::SIGNED,
- }
- }
-}
-
-impl fmt::Display for Type {
- fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
- write!(f, "{:?}", self)
- }
-}
-
-impl fmt::Display for ConvertedType {
- fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
- write!(f, "{:?}", self)
- }
-}
-
-impl fmt::Display for Repetition {
- fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
- write!(f, "{:?}", self)
- }
-}
-
-impl fmt::Display for Encoding {
- fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
- write!(f, "{:?}", self)
- }
-}
-
-impl fmt::Display for Compression {
- fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
- write!(f, "{:?}", self)
- }
-}
-
-impl fmt::Display for PageType {
- fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
- write!(f, "{:?}", self)
- }
-}
-
-impl fmt::Display for SortOrder {
- fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
- write!(f, "{:?}", self)
- }
-}
-
-impl fmt::Display for ColumnOrder {
- fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
- write!(f, "{:?}", self)
- }
-}
-
-// ----------------------------------------------------------------------
-// parquet::Type <=> Type conversion
-
-impl convert::From<parquet::Type> for Type {
- fn from(value: parquet::Type) -> Self {
- match value {
- parquet::Type::Boolean => Type::BOOLEAN,
- parquet::Type::Int32 => Type::INT32,
- parquet::Type::Int64 => Type::INT64,
- parquet::Type::Int96 => Type::INT96,
- parquet::Type::Float => Type::FLOAT,
- parquet::Type::Double => Type::DOUBLE,
- parquet::Type::ByteArray => Type::BYTE_ARRAY,
- parquet::Type::FixedLenByteArray => Type::FIXED_LEN_BYTE_ARRAY,
- }
- }
-}
-
-impl convert::From<Type> for parquet::Type {
- fn from(value: Type) -> Self {
- match value {
- Type::BOOLEAN => parquet::Type::Boolean,
- Type::INT32 => parquet::Type::Int32,
- Type::INT64 => parquet::Type::Int64,
- Type::INT96 => parquet::Type::Int96,
- Type::FLOAT => parquet::Type::Float,
- Type::DOUBLE => parquet::Type::Double,
- Type::BYTE_ARRAY => parquet::Type::ByteArray,
- Type::FIXED_LEN_BYTE_ARRAY => parquet::Type::FixedLenByteArray,
- }
- }
-}
-
-// ----------------------------------------------------------------------
-// parquet::ConvertedType <=> ConvertedType conversion
-
-impl convert::From<Option<parquet::ConvertedType>> for ConvertedType {
- fn from(option: Option<parquet::ConvertedType>) -> Self {
- match option {
- None => ConvertedType::NONE,
- Some(value) => match value {
- parquet::ConvertedType::Utf8 => ConvertedType::UTF8,
- parquet::ConvertedType::Map => ConvertedType::MAP,
- parquet::ConvertedType::MapKeyValue => ConvertedType::MAP_KEY_VALUE,
- parquet::ConvertedType::List => ConvertedType::LIST,
- parquet::ConvertedType::Enum => ConvertedType::ENUM,
- parquet::ConvertedType::Decimal => ConvertedType::DECIMAL,
- parquet::ConvertedType::Date => ConvertedType::DATE,
- parquet::ConvertedType::TimeMillis => ConvertedType::TIME_MILLIS,
- parquet::ConvertedType::TimeMicros => ConvertedType::TIME_MICROS,
- parquet::ConvertedType::TimestampMillis => {
- ConvertedType::TIMESTAMP_MILLIS
- }
- parquet::ConvertedType::TimestampMicros => {
- ConvertedType::TIMESTAMP_MICROS
- }
- parquet::ConvertedType::Uint8 => ConvertedType::UINT_8,
- parquet::ConvertedType::Uint16 => ConvertedType::UINT_16,
- parquet::ConvertedType::Uint32 => ConvertedType::UINT_32,
- parquet::ConvertedType::Uint64 => ConvertedType::UINT_64,
- parquet::ConvertedType::Int8 => ConvertedType::INT_8,
- parquet::ConvertedType::Int16 => ConvertedType::INT_16,
- parquet::ConvertedType::Int32 => ConvertedType::INT_32,
- parquet::ConvertedType::Int64 => ConvertedType::INT_64,
- parquet::ConvertedType::Json => ConvertedType::JSON,
- parquet::ConvertedType::Bson => ConvertedType::BSON,
- parquet::ConvertedType::Interval => ConvertedType::INTERVAL,
- },
- }
- }
-}
-
-impl convert::From<ConvertedType> for Option<parquet::ConvertedType> {
- fn from(value: ConvertedType) -> Self {
- match value {
- ConvertedType::NONE => None,
- ConvertedType::UTF8 => Some(parquet::ConvertedType::Utf8),
- ConvertedType::MAP => Some(parquet::ConvertedType::Map),
- ConvertedType::MAP_KEY_VALUE => Some(parquet::ConvertedType::MapKeyValue),
- ConvertedType::LIST => Some(parquet::ConvertedType::List),
- ConvertedType::ENUM => Some(parquet::ConvertedType::Enum),
- ConvertedType::DECIMAL => Some(parquet::ConvertedType::Decimal),
- ConvertedType::DATE => Some(parquet::ConvertedType::Date),
- ConvertedType::TIME_MILLIS => Some(parquet::ConvertedType::TimeMillis),
- ConvertedType::TIME_MICROS => Some(parquet::ConvertedType::TimeMicros),
- ConvertedType::TIMESTAMP_MILLIS => {
- Some(parquet::ConvertedType::TimestampMillis)
- }
- ConvertedType::TIMESTAMP_MICROS => {
- Some(parquet::ConvertedType::TimestampMicros)
- }
- ConvertedType::UINT_8 => Some(parquet::ConvertedType::Uint8),
- ConvertedType::UINT_16 => Some(parquet::ConvertedType::Uint16),
- ConvertedType::UINT_32 => Some(parquet::ConvertedType::Uint32),
- ConvertedType::UINT_64 => Some(parquet::ConvertedType::Uint64),
- ConvertedType::INT_8 => Some(parquet::ConvertedType::Int8),
- ConvertedType::INT_16 => Some(parquet::ConvertedType::Int16),
- ConvertedType::INT_32 => Some(parquet::ConvertedType::Int32),
- ConvertedType::INT_64 => Some(parquet::ConvertedType::Int64),
- ConvertedType::JSON => Some(parquet::ConvertedType::Json),
- ConvertedType::BSON => Some(parquet::ConvertedType::Bson),
- ConvertedType::INTERVAL => Some(parquet::ConvertedType::Interval),
- }
- }
-}
-
-// ----------------------------------------------------------------------
-// parquet::LogicalType <=> LogicalType conversion
-
-impl convert::From<parquet::LogicalType> for LogicalType {
- fn from(value: parquet::LogicalType) -> Self {
- match value {
- parquet::LogicalType::STRING(t) => LogicalType::STRING(t),
- parquet::LogicalType::MAP(t) => LogicalType::MAP(t),
- parquet::LogicalType::LIST(t) => LogicalType::LIST(t),
- parquet::LogicalType::ENUM(t) => LogicalType::ENUM(t),
- parquet::LogicalType::DECIMAL(t) => LogicalType::DECIMAL(t),
- parquet::LogicalType::DATE(t) => LogicalType::DATE(t),
- parquet::LogicalType::TIME(t) => LogicalType::TIME(t),
- parquet::LogicalType::TIMESTAMP(t) => LogicalType::TIMESTAMP(t),
- parquet::LogicalType::INTEGER(t) => LogicalType::INTEGER(t),
- parquet::LogicalType::UNKNOWN(t) => LogicalType::UNKNOWN(t),
- parquet::LogicalType::JSON(t) => LogicalType::JSON(t),
- parquet::LogicalType::BSON(t) => LogicalType::BSON(t),
- parquet::LogicalType::UUID(t) => LogicalType::UUID(t),
- }
- }
-}
-
-impl convert::From<LogicalType> for parquet::LogicalType {
- fn from(value: LogicalType) -> Self {
- match value {
- LogicalType::STRING(t) => parquet::LogicalType::STRING(t),
- LogicalType::MAP(t) => parquet::LogicalType::MAP(t),
- LogicalType::LIST(t) => parquet::LogicalType::LIST(t),
- LogicalType::ENUM(t) => parquet::LogicalType::ENUM(t),
- LogicalType::DECIMAL(t) => parquet::LogicalType::DECIMAL(t),
- LogicalType::DATE(t) => parquet::LogicalType::DATE(t),
- LogicalType::TIME(t) => parquet::LogicalType::TIME(t),
- LogicalType::TIMESTAMP(t) => parquet::LogicalType::TIMESTAMP(t),
- LogicalType::INTEGER(t) => parquet::LogicalType::INTEGER(t),
- LogicalType::UNKNOWN(t) => parquet::LogicalType::UNKNOWN(t),
- LogicalType::JSON(t) => parquet::LogicalType::JSON(t),
- LogicalType::BSON(t) => parquet::LogicalType::BSON(t),
- LogicalType::UUID(t) => parquet::LogicalType::UUID(t),
- }
- }
-}
-
-// ----------------------------------------------------------------------
-// LogicalType <=> ConvertedType conversion
-
-// Note: To prevent type loss when converting from ConvertedType to LogicalType,
-// the conversion from ConvertedType -> LogicalType is not implemented.
-// Such type loss includes:
-// - Not knowing the decimal scale and precision of ConvertedType
-// - Time and timestamp nanosecond precision, that is not supported in ConvertedType.
-
-impl From<Option<LogicalType>> for ConvertedType {
- fn from(value: Option<LogicalType>) -> Self {
- match value {
- Some(value) => match value {
- LogicalType::STRING(_) => ConvertedType::UTF8,
- LogicalType::MAP(_) => ConvertedType::MAP,
- LogicalType::LIST(_) => ConvertedType::LIST,
- LogicalType::ENUM(_) => ConvertedType::ENUM,
- LogicalType::DECIMAL(_) => ConvertedType::DECIMAL,
- LogicalType::DATE(_) => ConvertedType::DATE,
- LogicalType::TIME(t) => match t.unit {
- TimeUnit::MILLIS(_) => ConvertedType::TIME_MILLIS,
- TimeUnit::MICROS(_) => ConvertedType::TIME_MICROS,
- TimeUnit::NANOS(_) => ConvertedType::NONE,
- },
- LogicalType::TIMESTAMP(t) => match t.unit {
- TimeUnit::MILLIS(_) => ConvertedType::TIMESTAMP_MILLIS,
- TimeUnit::MICROS(_) => ConvertedType::TIMESTAMP_MICROS,
- TimeUnit::NANOS(_) => ConvertedType::NONE,
- },
- LogicalType::INTEGER(t) => match (t.bit_width, t.is_signed) {
- (8, true) => ConvertedType::INT_8,
- (16, true) => ConvertedType::INT_16,
- (32, true) => ConvertedType::INT_32,
- (64, true) => ConvertedType::INT_64,
- (8, false) => ConvertedType::UINT_8,
- (16, false) => ConvertedType::UINT_16,
- (32, false) => ConvertedType::UINT_32,
- (64, false) => ConvertedType::UINT_64,
- t => panic!("Integer type {:?} is not supported", t),
- },
- LogicalType::UNKNOWN(_) => ConvertedType::NONE,
- LogicalType::JSON(_) => ConvertedType::JSON,
- LogicalType::BSON(_) => ConvertedType::BSON,
- LogicalType::UUID(_) => ConvertedType::NONE,
- },
- None => ConvertedType::NONE,
- }
- }
-}
-
-// ----------------------------------------------------------------------
-// parquet::FieldRepetitionType <=> Repetition conversion
-
-impl convert::From<parquet::FieldRepetitionType> for Repetition {
- fn from(value: parquet::FieldRepetitionType) -> Self {
- match value {
- parquet::FieldRepetitionType::Required => Repetition::REQUIRED,
- parquet::FieldRepetitionType::Optional => Repetition::OPTIONAL,
- parquet::FieldRepetitionType::Repeated => Repetition::REPEATED,
- }
- }
-}
-
-impl convert::From<Repetition> for parquet::FieldRepetitionType {
- fn from(value: Repetition) -> Self {
- match value {
- Repetition::REQUIRED => parquet::FieldRepetitionType::Required,
- Repetition::OPTIONAL => parquet::FieldRepetitionType::Optional,
- Repetition::REPEATED => parquet::FieldRepetitionType::Repeated,
- }
- }
-}
-
-// ----------------------------------------------------------------------
-// parquet::Encoding <=> Encoding conversion
-
-impl convert::From<parquet::Encoding> for Encoding {
- fn from(value: parquet::Encoding) -> Self {
- match value {
- parquet::Encoding::Plain => Encoding::PLAIN,
- parquet::Encoding::PlainDictionary => Encoding::PLAIN_DICTIONARY,
- parquet::Encoding::Rle => Encoding::RLE,
- parquet::Encoding::BitPacked => Encoding::BIT_PACKED,
- parquet::Encoding::DeltaBinaryPacked => Encoding::DELTA_BINARY_PACKED,
- parquet::Encoding::DeltaLengthByteArray => Encoding::DELTA_LENGTH_BYTE_ARRAY,
- parquet::Encoding::DeltaByteArray => Encoding::DELTA_BYTE_ARRAY,
- parquet::Encoding::RleDictionary => Encoding::RLE_DICTIONARY,
- }
- }
-}
-
-impl convert::From<Encoding> for parquet::Encoding {
- fn from(value: Encoding) -> Self {
- match value {
- Encoding::PLAIN => parquet::Encoding::Plain,
- Encoding::PLAIN_DICTIONARY => parquet::Encoding::PlainDictionary,
- Encoding::RLE => parquet::Encoding::Rle,
- Encoding::BIT_PACKED => parquet::Encoding::BitPacked,
- Encoding::DELTA_BINARY_PACKED => parquet::Encoding::DeltaBinaryPacked,
- Encoding::DELTA_LENGTH_BYTE_ARRAY => parquet::Encoding::DeltaLengthByteArray,
- Encoding::DELTA_BYTE_ARRAY => parquet::Encoding::DeltaByteArray,
- Encoding::RLE_DICTIONARY => parquet::Encoding::RleDictionary,
- }
- }
-}
-
-// ----------------------------------------------------------------------
-// parquet::CompressionCodec <=> Compression conversion
-
-impl convert::From<parquet::CompressionCodec> for Compression {
- fn from(value: parquet::CompressionCodec) -> Self {
- match value {
- parquet::CompressionCodec::Uncompressed => Compression::UNCOMPRESSED,
- parquet::CompressionCodec::Snappy => Compression::SNAPPY,
- parquet::CompressionCodec::Gzip => Compression::GZIP,
- parquet::CompressionCodec::Lzo => Compression::LZO,
- parquet::CompressionCodec::Brotli => Compression::BROTLI,
- parquet::CompressionCodec::Lz4 => Compression::LZ4,
- parquet::CompressionCodec::Zstd => Compression::ZSTD,
- }
- }
-}
-
-impl convert::From<Compression> for parquet::CompressionCodec {
- fn from(value: Compression) -> Self {
- match value {
- Compression::UNCOMPRESSED => parquet::CompressionCodec::Uncompressed,
- Compression::SNAPPY => parquet::CompressionCodec::Snappy,
- Compression::GZIP => parquet::CompressionCodec::Gzip,
- Compression::LZO => parquet::CompressionCodec::Lzo,
- Compression::BROTLI => parquet::CompressionCodec::Brotli,
- Compression::LZ4 => parquet::CompressionCodec::Lz4,
- Compression::ZSTD => parquet::CompressionCodec::Zstd,
- }
- }
-}
-
-// ----------------------------------------------------------------------
-// parquet::PageType <=> PageType conversion
-
-impl convert::From<parquet::PageType> for PageType {
- fn from(value: parquet::PageType) -> Self {
- match value {
- parquet::PageType::DataPage => PageType::DATA_PAGE,
- parquet::PageType::IndexPage => PageType::INDEX_PAGE,
- parquet::PageType::DictionaryPage => PageType::DICTIONARY_PAGE,
- parquet::PageType::DataPageV2 => PageType::DATA_PAGE_V2,
- }
- }
-}
-
-impl convert::From<PageType> for parquet::PageType {
- fn from(value: PageType) -> Self {
- match value {
- PageType::DATA_PAGE => parquet::PageType::DataPage,
- PageType::INDEX_PAGE => parquet::PageType::IndexPage,
- PageType::DICTIONARY_PAGE => parquet::PageType::DictionaryPage,
- PageType::DATA_PAGE_V2 => parquet::PageType::DataPageV2,
- }
- }
-}
-
-// ----------------------------------------------------------------------
-// String conversions for schema parsing.
-
-impl str::FromStr for Repetition {
- type Err = ParquetError;
-
- fn from_str(s: &str) -> result::Result<Self, Self::Err> {
- match s {
- "REQUIRED" => Ok(Repetition::REQUIRED),
- "OPTIONAL" => Ok(Repetition::OPTIONAL),
- "REPEATED" => Ok(Repetition::REPEATED),
- other => Err(general_err!("Invalid repetition {}", other)),
- }
- }
-}
-
-impl str::FromStr for Type {
- type Err = ParquetError;
-
- fn from_str(s: &str) -> result::Result<Self, Self::Err> {
- match s {
- "BOOLEAN" => Ok(Type::BOOLEAN),
- "INT32" => Ok(Type::INT32),
- "INT64" => Ok(Type::INT64),
- "INT96" => Ok(Type::INT96),
- "FLOAT" => Ok(Type::FLOAT),
- "DOUBLE" => Ok(Type::DOUBLE),
- "BYTE_ARRAY" | "BINARY" => Ok(Type::BYTE_ARRAY),
- "FIXED_LEN_BYTE_ARRAY" => Ok(Type::FIXED_LEN_BYTE_ARRAY),
- other => Err(general_err!("Invalid type {}", other)),
- }
- }
-}
-
-impl str::FromStr for ConvertedType {
- type Err = ParquetError;
-
- fn from_str(s: &str) -> result::Result<Self, Self::Err> {
- match s {
- "NONE" => Ok(ConvertedType::NONE),
- "UTF8" => Ok(ConvertedType::UTF8),
- "MAP" => Ok(ConvertedType::MAP),
- "MAP_KEY_VALUE" => Ok(ConvertedType::MAP_KEY_VALUE),
- "LIST" => Ok(ConvertedType::LIST),
- "ENUM" => Ok(ConvertedType::ENUM),
- "DECIMAL" => Ok(ConvertedType::DECIMAL),
- "DATE" => Ok(ConvertedType::DATE),
- "TIME_MILLIS" => Ok(ConvertedType::TIME_MILLIS),
- "TIME_MICROS" => Ok(ConvertedType::TIME_MICROS),
- "TIMESTAMP_MILLIS" => Ok(ConvertedType::TIMESTAMP_MILLIS),
- "TIMESTAMP_MICROS" => Ok(ConvertedType::TIMESTAMP_MICROS),
- "UINT_8" => Ok(ConvertedType::UINT_8),
- "UINT_16" => Ok(ConvertedType::UINT_16),
- "UINT_32" => Ok(ConvertedType::UINT_32),
- "UINT_64" => Ok(ConvertedType::UINT_64),
- "INT_8" => Ok(ConvertedType::INT_8),
- "INT_16" => Ok(ConvertedType::INT_16),
- "INT_32" => Ok(ConvertedType::INT_32),
- "INT_64" => Ok(ConvertedType::INT_64),
- "JSON" => Ok(ConvertedType::JSON),
- "BSON" => Ok(ConvertedType::BSON),
- "INTERVAL" => Ok(ConvertedType::INTERVAL),
- other => Err(general_err!("Invalid converted type {}", other)),
- }
- }
-}
-
-impl str::FromStr for LogicalType {
- type Err = ParquetError;
-
- fn from_str(s: &str) -> result::Result<Self, Self::Err> {
- match s {
- // The type is a placeholder that gets updated elsewhere
- "INTEGER" => Ok(LogicalType::INTEGER(IntType {
- bit_width: 8,
- is_signed: false,
- })),
- "MAP" => Ok(LogicalType::MAP(MapType {})),
- "LIST" => Ok(LogicalType::LIST(ListType {})),
- "ENUM" => Ok(LogicalType::ENUM(EnumType {})),
- "DECIMAL" => Ok(LogicalType::DECIMAL(DecimalType {
- precision: -1,
- scale: -1,
- })),
- "DATE" => Ok(LogicalType::DATE(DateType {})),
- "TIME" => Ok(LogicalType::TIME(TimeType {
- is_adjusted_to_u_t_c: false,
- unit: TimeUnit::MILLIS(parquet::MilliSeconds {}),
- })),
- "TIMESTAMP" => Ok(LogicalType::TIMESTAMP(TimestampType {
- is_adjusted_to_u_t_c: false,
- unit: TimeUnit::MILLIS(parquet::MilliSeconds {}),
- })),
- "STRING" => Ok(LogicalType::STRING(StringType {})),
- "JSON" => Ok(LogicalType::JSON(JsonType {})),
- "BSON" => Ok(LogicalType::BSON(BsonType {})),
- "UUID" => Ok(LogicalType::UUID(UUIDType {})),
- "UNKNOWN" => Ok(LogicalType::UNKNOWN(NullType {})),
- "INTERVAL" => Err(general_err!("Interval logical type not yet supported")),
- other => Err(general_err!("Invalid logical type {}", other)),
- }
- }
-}
-
-#[cfg(test)]
-mod tests {
- use super::*;
-
- #[test]
- fn test_display_type() {
- assert_eq!(Type::BOOLEAN.to_string(), "BOOLEAN");
- assert_eq!(Type::INT32.to_string(), "INT32");
- assert_eq!(Type::INT64.to_string(), "INT64");
- assert_eq!(Type::INT96.to_string(), "INT96");
- assert_eq!(Type::FLOAT.to_string(), "FLOAT");
- assert_eq!(Type::DOUBLE.to_string(), "DOUBLE");
- assert_eq!(Type::BYTE_ARRAY.to_string(), "BYTE_ARRAY");
- assert_eq!(
- Type::FIXED_LEN_BYTE_ARRAY.to_string(),
- "FIXED_LEN_BYTE_ARRAY"
- );
- }
-
- #[test]
- fn test_from_type() {
- assert_eq!(Type::from(parquet::Type::Boolean), Type::BOOLEAN);
- assert_eq!(Type::from(parquet::Type::Int32), Type::INT32);
- assert_eq!(Type::from(parquet::Type::Int64), Type::INT64);
- assert_eq!(Type::from(parquet::Type::Int96), Type::INT96);
- assert_eq!(Type::from(parquet::Type::Float), Type::FLOAT);
- assert_eq!(Type::from(parquet::Type::Double), Type::DOUBLE);
- assert_eq!(Type::from(parquet::Type::ByteArray), Type::BYTE_ARRAY);
- assert_eq!(
- Type::from(parquet::Type::FixedLenByteArray),
- Type::FIXED_LEN_BYTE_ARRAY
- );
- }
-
- #[test]
- fn test_into_type() {
- assert_eq!(parquet::Type::Boolean, Type::BOOLEAN.into());
- assert_eq!(parquet::Type::Int32, Type::INT32.into());
- assert_eq!(parquet::Type::Int64, Type::INT64.into());
- assert_eq!(parquet::Type::Int96, Type::INT96.into());
- assert_eq!(parquet::Type::Float, Type::FLOAT.into());
- assert_eq!(parquet::Type::Double, Type::DOUBLE.into());
- assert_eq!(parquet::Type::ByteArray, Type::BYTE_ARRAY.into());
- assert_eq!(
- parquet::Type::FixedLenByteArray,
- Type::FIXED_LEN_BYTE_ARRAY.into()
- );
- }
-
- #[test]
- fn test_from_string_into_type() {
- assert_eq!(
- Type::BOOLEAN.to_string().parse::<Type>().unwrap(),
- Type::BOOLEAN
- );
- assert_eq!(
- Type::INT32.to_string().parse::<Type>().unwrap(),
- Type::INT32
- );
- assert_eq!(
- Type::INT64.to_string().parse::<Type>().unwrap(),
- Type::INT64
- );
- assert_eq!(
- Type::INT96.to_string().parse::<Type>().unwrap(),
- Type::INT96
- );
- assert_eq!(
- Type::FLOAT.to_string().parse::<Type>().unwrap(),
- Type::FLOAT
- );
- assert_eq!(
- Type::DOUBLE.to_string().parse::<Type>().unwrap(),
- Type::DOUBLE
- );
- assert_eq!(
- Type::BYTE_ARRAY.to_string().parse::<Type>().unwrap(),
- Type::BYTE_ARRAY
- );
- assert_eq!("BINARY".parse::<Type>().unwrap(), Type::BYTE_ARRAY);
- assert_eq!(
- Type::FIXED_LEN_BYTE_ARRAY
- .to_string()
- .parse::<Type>()
- .unwrap(),
- Type::FIXED_LEN_BYTE_ARRAY
- );
- }
-
- #[test]
- fn test_display_converted_type() {
- assert_eq!(ConvertedType::NONE.to_string(), "NONE");
- assert_eq!(ConvertedType::UTF8.to_string(), "UTF8");
- assert_eq!(ConvertedType::MAP.to_string(), "MAP");
- assert_eq!(ConvertedType::MAP_KEY_VALUE.to_string(), "MAP_KEY_VALUE");
- assert_eq!(ConvertedType::LIST.to_string(), "LIST");
- assert_eq!(ConvertedType::ENUM.to_string(), "ENUM");
- assert_eq!(ConvertedType::DECIMAL.to_string(), "DECIMAL");
- assert_eq!(ConvertedType::DATE.to_string(), "DATE");
- assert_eq!(ConvertedType::TIME_MILLIS.to_string(), "TIME_MILLIS");
- assert_eq!(ConvertedType::DATE.to_string(), "DATE");
- assert_eq!(ConvertedType::TIME_MICROS.to_string(), "TIME_MICROS");
- assert_eq!(
- ConvertedType::TIMESTAMP_MILLIS.to_string(),
- "TIMESTAMP_MILLIS"
- );
- assert_eq!(
- ConvertedType::TIMESTAMP_MICROS.to_string(),
- "TIMESTAMP_MICROS"
- );
- assert_eq!(ConvertedType::UINT_8.to_string(), "UINT_8");
- assert_eq!(ConvertedType::UINT_16.to_string(), "UINT_16");
- assert_eq!(ConvertedType::UINT_32.to_string(), "UINT_32");
- assert_eq!(ConvertedType::UINT_64.to_string(), "UINT_64");
- assert_eq!(ConvertedType::INT_8.to_string(), "INT_8");
- assert_eq!(ConvertedType::INT_16.to_string(), "INT_16");
- assert_eq!(ConvertedType::INT_32.to_string(), "INT_32");
- assert_eq!(ConvertedType::INT_64.to_string(), "INT_64");
- assert_eq!(ConvertedType::JSON.to_string(), "JSON");
- assert_eq!(ConvertedType::BSON.to_string(), "BSON");
- assert_eq!(ConvertedType::INTERVAL.to_string(), "INTERVAL");
- }
-
- #[test]
- fn test_from_converted_type() {
- let parquet_conv_none: Option<parquet::ConvertedType> = None;
- assert_eq!(ConvertedType::from(parquet_conv_none), ConvertedType::NONE);
- assert_eq!(
- ConvertedType::from(Some(parquet::ConvertedType::Utf8)),
- ConvertedType::UTF8
- );
- assert_eq!(
- ConvertedType::from(Some(parquet::ConvertedType::Map)),
- ConvertedType::MAP
- );
- assert_eq!(
- ConvertedType::from(Some(parquet::ConvertedType::MapKeyValue)),
- ConvertedType::MAP_KEY_VALUE
- );
- assert_eq!(
- ConvertedType::from(Some(parquet::ConvertedType::List)),
- ConvertedType::LIST
- );
- assert_eq!(
- ConvertedType::from(Some(parquet::ConvertedType::Enum)),
- ConvertedType::ENUM
- );
- assert_eq!(
- ConvertedType::from(Some(parquet::ConvertedType::Decimal)),
- ConvertedType::DECIMAL
- );
- assert_eq!(
- ConvertedType::from(Some(parquet::ConvertedType::Date)),
- ConvertedType::DATE
- );
- assert_eq!(
- ConvertedType::from(Some(parquet::ConvertedType::TimeMillis)),
- ConvertedType::TIME_MILLIS
- );
- assert_eq!(
- ConvertedType::from(Some(parquet::ConvertedType::TimeMicros)),
- ConvertedType::TIME_MICROS
- );
- assert_eq!(
- ConvertedType::from(Some(parquet::ConvertedType::TimestampMillis)),
- ConvertedType::TIMESTAMP_MILLIS
- );
- assert_eq!(
- ConvertedType::from(Some(parquet::ConvertedType::TimestampMicros)),
- ConvertedType::TIMESTAMP_MICROS
- );
- assert_eq!(
- ConvertedType::from(Some(parquet::ConvertedType::Uint8)),
- ConvertedType::UINT_8
- );
- assert_eq!(
- ConvertedType::from(Some(parquet::ConvertedType::Uint16)),
- ConvertedType::UINT_16
- );
- assert_eq!(
- ConvertedType::from(Some(parquet::ConvertedType::Uint32)),
- ConvertedType::UINT_32
- );
- assert_eq!(
- ConvertedType::from(Some(parquet::ConvertedType::Uint64)),
- ConvertedType::UINT_64
- );
- assert_eq!(
- ConvertedType::from(Some(parquet::ConvertedType::Int8)),
- ConvertedType::INT_8
- );
- assert_eq!(
- ConvertedType::from(Some(parquet::ConvertedType::Int16)),
- ConvertedType::INT_16
- );
- assert_eq!(
- ConvertedType::from(Some(parquet::ConvertedType::Int32)),
- ConvertedType::INT_32
- );
- assert_eq!(
- ConvertedType::from(Some(parquet::ConvertedType::Int64)),
- ConvertedType::INT_64
- );
- assert_eq!(
- ConvertedType::from(Some(parquet::ConvertedType::Json)),
- ConvertedType::JSON
- );
- assert_eq!(
- ConvertedType::from(Some(parquet::ConvertedType::Bson)),
- ConvertedType::BSON
- );
- assert_eq!(
- ConvertedType::from(Some(parquet::ConvertedType::Interval)),
- ConvertedType::INTERVAL
- );
- }
-
- #[test]
- fn test_into_converted_type() {
- let converted_type: Option<parquet::ConvertedType> = None;
- assert_eq!(converted_type, ConvertedType::NONE.into());
- assert_eq!(
- Some(parquet::ConvertedType::Utf8),
- ConvertedType::UTF8.into()
- );
- assert_eq!(Some(parquet::ConvertedType::Map), ConvertedType::MAP.into());
- assert_eq!(
- Some(parquet::ConvertedType::MapKeyValue),
- ConvertedType::MAP_KEY_VALUE.into()
- );
- assert_eq!(
- Some(parquet::ConvertedType::List),
- ConvertedType::LIST.into()
- );
- assert_eq!(
- Some(parquet::ConvertedType::Enum),
- ConvertedType::ENUM.into()
- );
- assert_eq!(
- Some(parquet::ConvertedType::Decimal),
- ConvertedType::DECIMAL.into()
- );
- assert_eq!(
- Some(parquet::ConvertedType::Date),
- ConvertedType::DATE.into()
- );
- assert_eq!(
- Some(parquet::ConvertedType::TimeMillis),
- ConvertedType::TIME_MILLIS.into()
- );
- assert_eq!(
- Some(parquet::ConvertedType::TimeMicros),
- ConvertedType::TIME_MICROS.into()
- );
- assert_eq!(
- Some(parquet::ConvertedType::TimestampMillis),
- ConvertedType::TIMESTAMP_MILLIS.into()
- );
- assert_eq!(
- Some(parquet::ConvertedType::TimestampMicros),
- ConvertedType::TIMESTAMP_MICROS.into()
- );
- assert_eq!(
- Some(parquet::ConvertedType::Uint8),
- ConvertedType::UINT_8.into()
- );
- assert_eq!(
- Some(parquet::ConvertedType::Uint16),
- ConvertedType::UINT_16.into()
- );
- assert_eq!(
- Some(parquet::ConvertedType::Uint32),
- ConvertedType::UINT_32.into()
- );
- assert_eq!(
- Some(parquet::ConvertedType::Uint64),
- ConvertedType::UINT_64.into()
- );
- assert_eq!(
- Some(parquet::ConvertedType::Int8),
- ConvertedType::INT_8.into()
- );
- assert_eq!(
- Some(parquet::ConvertedType::Int16),
- ConvertedType::INT_16.into()
- );
- assert_eq!(
- Some(parquet::ConvertedType::Int32),
- ConvertedType::INT_32.into()
- );
- assert_eq!(
- Some(parquet::ConvertedType::Int64),
- ConvertedType::INT_64.into()
- );
- assert_eq!(
- Some(parquet::ConvertedType::Json),
- ConvertedType::JSON.into()
- );
- assert_eq!(
- Some(parquet::ConvertedType::Bson),
- ConvertedType::BSON.into()
- );
- assert_eq!(
- Some(parquet::ConvertedType::Interval),
- ConvertedType::INTERVAL.into()
- );
- }
-
- #[test]
- fn test_from_string_into_converted_type() {
- assert_eq!(
- ConvertedType::NONE
- .to_string()
- .parse::<ConvertedType>()
- .unwrap(),
- ConvertedType::NONE
- );
- assert_eq!(
- ConvertedType::UTF8
- .to_string()
- .parse::<ConvertedType>()
- .unwrap(),
- ConvertedType::UTF8
- );
- assert_eq!(
- ConvertedType::MAP
- .to_string()
- .parse::<ConvertedType>()
- .unwrap(),
- ConvertedType::MAP
- );
- assert_eq!(
- ConvertedType::MAP_KEY_VALUE
- .to_string()
- .parse::<ConvertedType>()
- .unwrap(),
- ConvertedType::MAP_KEY_VALUE
- );
- assert_eq!(
- ConvertedType::LIST
- .to_string()
- .parse::<ConvertedType>()
- .unwrap(),
- ConvertedType::LIST
- );
- assert_eq!(
- ConvertedType::ENUM
- .to_string()
- .parse::<ConvertedType>()
- .unwrap(),
- ConvertedType::ENUM
- );
- assert_eq!(
- ConvertedType::DECIMAL
- .to_string()
- .parse::<ConvertedType>()
- .unwrap(),
- ConvertedType::DECIMAL
- );
- assert_eq!(
- ConvertedType::DATE
- .to_string()
- .parse::<ConvertedType>()
- .unwrap(),
- ConvertedType::DATE
- );
- assert_eq!(
- ConvertedType::TIME_MILLIS
- .to_string()
- .parse::<ConvertedType>()
- .unwrap(),
- ConvertedType::TIME_MILLIS
- );
- assert_eq!(
- ConvertedType::TIME_MICROS
- .to_string()
- .parse::<ConvertedType>()
- .unwrap(),
- ConvertedType::TIME_MICROS
- );
- assert_eq!(
- ConvertedType::TIMESTAMP_MILLIS
- .to_string()
- .parse::<ConvertedType>()
- .unwrap(),
- ConvertedType::TIMESTAMP_MILLIS
- );
- assert_eq!(
- ConvertedType::TIMESTAMP_MICROS
- .to_string()
- .parse::<ConvertedType>()
- .unwrap(),
- ConvertedType::TIMESTAMP_MICROS
- );
- assert_eq!(
- ConvertedType::UINT_8
- .to_string()
- .parse::<ConvertedType>()
- .unwrap(),
- ConvertedType::UINT_8
- );
- assert_eq!(
- ConvertedType::UINT_16
- .to_string()
- .parse::<ConvertedType>()
- .unwrap(),
- ConvertedType::UINT_16
- );
- assert_eq!(
- ConvertedType::UINT_32
- .to_string()
- .parse::<ConvertedType>()
- .unwrap(),
- ConvertedType::UINT_32
- );
- assert_eq!(
- ConvertedType::UINT_64
- .to_string()
- .parse::<ConvertedType>()
- .unwrap(),
- ConvertedType::UINT_64
- );
- assert_eq!(
- ConvertedType::INT_8
- .to_string()
- .parse::<ConvertedType>()
- .unwrap(),
- ConvertedType::INT_8
- );
- assert_eq!(
- ConvertedType::INT_16
- .to_string()
- .parse::<ConvertedType>()
- .unwrap(),
- ConvertedType::INT_16
- );
- assert_eq!(
- ConvertedType::INT_32
- .to_string()
- .parse::<ConvertedType>()
- .unwrap(),
- ConvertedType::INT_32
- );
- assert_eq!(
- ConvertedType::INT_64
- .to_string()
- .parse::<ConvertedType>()
- .unwrap(),
- ConvertedType::INT_64
- );
- assert_eq!(
- ConvertedType::JSON
- .to_string()
- .parse::<ConvertedType>()
- .unwrap(),
- ConvertedType::JSON
- );
- assert_eq!(
- ConvertedType::BSON
- .to_string()
- .parse::<ConvertedType>()
- .unwrap(),
- ConvertedType::BSON
- );
- assert_eq!(
- ConvertedType::INTERVAL
- .to_string()
- .parse::<ConvertedType>()
- .unwrap(),
- ConvertedType::INTERVAL
- );
- }
-
- #[test]
- fn test_logical_to_converted_type() {
- let logical_none: Option<LogicalType> = None;
- assert_eq!(ConvertedType::from(logical_none), ConvertedType::NONE);
- assert_eq!(
- ConvertedType::from(Some(LogicalType::DECIMAL(DecimalType {
- precision: 20,
- scale: 5
- }))),
- ConvertedType::DECIMAL
- );
- assert_eq!(
- ConvertedType::from(Some(LogicalType::BSON(Default::default()))),
- ConvertedType::BSON
- );
- assert_eq!(
- ConvertedType::from(Some(LogicalType::JSON(Default::default()))),
- ConvertedType::JSON
- );
- assert_eq!(
- ConvertedType::from(Some(LogicalType::STRING(Default::default()))),
- ConvertedType::UTF8
- );
- assert_eq!(
- ConvertedType::from(Some(LogicalType::DATE(Default::default()))),
- ConvertedType::DATE
- );
- assert_eq!(
- ConvertedType::from(Some(LogicalType::TIME(TimeType {
- unit: TimeUnit::MILLIS(Default::default()),
- is_adjusted_to_u_t_c: true,
- }))),
- ConvertedType::TIME_MILLIS
- );
- assert_eq!(
- ConvertedType::from(Some(LogicalType::TIME(TimeType {
- unit: TimeUnit::MICROS(Default::default()),
- is_adjusted_to_u_t_c: true,
- }))),
- ConvertedType::TIME_MICROS
- );
- assert_eq!(
- ConvertedType::from(Some(LogicalType::TIME(TimeType {
- unit: TimeUnit::NANOS(Default::default()),
- is_adjusted_to_u_t_c: false,
- }))),
- ConvertedType::NONE
- );
- assert_eq!(
- ConvertedType::from(Some(LogicalType::TIMESTAMP(TimestampType {
- unit: TimeUnit::MILLIS(Default::default()),
- is_adjusted_to_u_t_c: true,
- }))),
- ConvertedType::TIMESTAMP_MILLIS
- );
- assert_eq!(
- ConvertedType::from(Some(LogicalType::TIMESTAMP(TimestampType {
- unit: TimeUnit::MICROS(Default::default()),
- is_adjusted_to_u_t_c: false,
- }))),
- ConvertedType::TIMESTAMP_MICROS
- );
- assert_eq!(
- ConvertedType::from(Some(LogicalType::TIMESTAMP(TimestampType {
- unit: TimeUnit::NANOS(Default::default()),
- is_adjusted_to_u_t_c: false,
- }))),
- ConvertedType::NONE
- );
- assert_eq!(
- ConvertedType::from(Some(LogicalType::INTEGER(IntType {
- bit_width: 8,
- is_signed: false
- }))),
- ConvertedType::UINT_8
- );
- assert_eq!(
- ConvertedType::from(Some(LogicalType::INTEGER(IntType {
- bit_width: 8,
- is_signed: true
- }))),
- ConvertedType::INT_8
- );
- assert_eq!(
- ConvertedType::from(Some(LogicalType::INTEGER(IntType {
- bit_width: 16,
- is_signed: false
- }))),
- ConvertedType::UINT_16
- );
- assert_eq!(
- ConvertedType::from(Some(LogicalType::INTEGER(IntType {
- bit_width: 16,
- is_signed: true
- }))),
- ConvertedType::INT_16
- );
- assert_eq!(
- ConvertedType::from(Some(LogicalType::INTEGER(IntType {
- bit_width: 32,
- is_signed: false
- }))),
- ConvertedType::UINT_32
- );
- assert_eq!(
- ConvertedType::from(Some(LogicalType::INTEGER(IntType {
- bit_width: 32,
- is_signed: true
- }))),
- ConvertedType::INT_32
- );
- assert_eq!(
- ConvertedType::from(Some(LogicalType::INTEGER(IntType {
- bit_width: 64,
- is_signed: false
- }))),
- ConvertedType::UINT_64
- );
- assert_eq!(
- ConvertedType::from(Some(LogicalType::INTEGER(IntType {
- bit_width: 64,
- is_signed: true
- }))),
- ConvertedType::INT_64
- );
- assert_eq!(
- ConvertedType::from(Some(LogicalType::LIST(Default::default()))),
- ConvertedType::LIST
- );
- assert_eq!(
- ConvertedType::from(Some(LogicalType::MAP(Default::default()))),
- ConvertedType::MAP
- );
- assert_eq!(
- ConvertedType::from(Some(LogicalType::UUID(Default::default()))),
- ConvertedType::NONE
- );
- assert_eq!(
- ConvertedType::from(Some(LogicalType::ENUM(Default::default()))),
- ConvertedType::ENUM
- );
- assert_eq!(
- ConvertedType::from(Some(LogicalType::UNKNOWN(Default::default()))),
- ConvertedType::NONE
- );
- }
-
- #[test]
- fn test_display_repetition() {
- assert_eq!(Repetition::REQUIRED.to_string(), "REQUIRED");
- assert_eq!(Repetition::OPTIONAL.to_string(), "OPTIONAL");
- assert_eq!(Repetition::REPEATED.to_string(), "REPEATED");
- }
-
- #[test]
- fn test_from_repetition() {
- assert_eq!(
- Repetition::from(parquet::FieldRepetitionType::Required),
- Repetition::REQUIRED
- );
- assert_eq!(
- Repetition::from(parquet::FieldRepetitionType::Optional),
- Repetition::OPTIONAL
- );
- assert_eq!(
- Repetition::from(parquet::FieldRepetitionType::Repeated),
- Repetition::REPEATED
- );
- }
-
- #[test]
- fn test_into_repetition() {
- assert_eq!(
- parquet::FieldRepetitionType::Required,
- Repetition::REQUIRED.into()
- );
- assert_eq!(
- parquet::FieldRepetitionType::Optional,
- Repetition::OPTIONAL.into()
- );
- assert_eq!(
- parquet::FieldRepetitionType::Repeated,
- Repetition::REPEATED.into()
- );
- }
-
- #[test]
- fn test_from_string_into_repetition() {
- assert_eq!(
- Repetition::REQUIRED
- .to_string()
- .parse::<Repetition>()
- .unwrap(),
- Repetition::REQUIRED
- );
- assert_eq!(
- Repetition::OPTIONAL
- .to_string()
- .parse::<Repetition>()
- .unwrap(),
- Repetition::OPTIONAL
- );
- assert_eq!(
- Repetition::REPEATED
- .to_string()
- .parse::<Repetition>()
- .unwrap(),
- Repetition::REPEATED
- );
- }
-
- #[test]
- fn test_display_encoding() {
- assert_eq!(Encoding::PLAIN.to_string(), "PLAIN");
- assert_eq!(Encoding::PLAIN_DICTIONARY.to_string(), "PLAIN_DICTIONARY");
- assert_eq!(Encoding::RLE.to_string(), "RLE");
- assert_eq!(Encoding::BIT_PACKED.to_string(), "BIT_PACKED");
- assert_eq!(
- Encoding::DELTA_BINARY_PACKED.to_string(),
- "DELTA_BINARY_PACKED"
- );
- assert_eq!(
- Encoding::DELTA_LENGTH_BYTE_ARRAY.to_string(),
- "DELTA_LENGTH_BYTE_ARRAY"
- );
- assert_eq!(Encoding::DELTA_BYTE_ARRAY.to_string(), "DELTA_BYTE_ARRAY");
- assert_eq!(Encoding::RLE_DICTIONARY.to_string(), "RLE_DICTIONARY");
- }
-
- #[test]
- fn test_from_encoding() {
- assert_eq!(Encoding::from(parquet::Encoding::Plain), Encoding::PLAIN);
- assert_eq!(
- Encoding::from(parquet::Encoding::PlainDictionary),
- Encoding::PLAIN_DICTIONARY
- );
- assert_eq!(Encoding::from(parquet::Encoding::Rle), Encoding::RLE);
- assert_eq!(
- Encoding::from(parquet::Encoding::BitPacked),
- Encoding::BIT_PACKED
- );
- assert_eq!(
- Encoding::from(parquet::Encoding::DeltaBinaryPacked),
- Encoding::DELTA_BINARY_PACKED
- );
- assert_eq!(
- Encoding::from(parquet::Encoding::DeltaLengthByteArray),
- Encoding::DELTA_LENGTH_BYTE_ARRAY
- );
- assert_eq!(
- Encoding::from(parquet::Encoding::DeltaByteArray),
- Encoding::DELTA_BYTE_ARRAY
- );
- }
-
- #[test]
- fn test_into_encoding() {
- assert_eq!(parquet::Encoding::Plain, Encoding::PLAIN.into());
- assert_eq!(
- parquet::Encoding::PlainDictionary,
- Encoding::PLAIN_DICTIONARY.into()
- );
- assert_eq!(parquet::Encoding::Rle, Encoding::RLE.into());
- assert_eq!(parquet::Encoding::BitPacked, Encoding::BIT_PACKED.into());
- assert_eq!(
- parquet::Encoding::DeltaBinaryPacked,
- Encoding::DELTA_BINARY_PACKED.into()
- );
- assert_eq!(
- parquet::Encoding::DeltaLengthByteArray,
- Encoding::DELTA_LENGTH_BYTE_ARRAY.into()
- );
- assert_eq!(
- parquet::Encoding::DeltaByteArray,
- Encoding::DELTA_BYTE_ARRAY.into()
- );
- }
-
- #[test]
- fn test_display_compression() {
- assert_eq!(Compression::UNCOMPRESSED.to_string(), "UNCOMPRESSED");
- assert_eq!(Compression::SNAPPY.to_string(), "SNAPPY");
- assert_eq!(Compression::GZIP.to_string(), "GZIP");
- assert_eq!(Compression::LZO.to_string(), "LZO");
- assert_eq!(Compression::BROTLI.to_string(), "BROTLI");
- assert_eq!(Compression::LZ4.to_string(), "LZ4");
- assert_eq!(Compression::ZSTD.to_string(), "ZSTD");
- }
-
- #[test]
- fn test_from_compression() {
- assert_eq!(
- Compression::from(parquet::CompressionCodec::Uncompressed),
- Compression::UNCOMPRESSED
- );
- assert_eq!(
- Compression::from(parquet::CompressionCodec::Snappy),
- Compression::SNAPPY
- );
- assert_eq!(
- Compression::from(parquet::CompressionCodec::Gzip),
- Compression::GZIP
- );
- assert_eq!(
- Compression::from(parquet::CompressionCodec::Lzo),
- Compression::LZO
- );
- assert_eq!(
- Compression::from(parquet::CompressionCodec::Brotli),
- Compression::BROTLI
- );
- assert_eq!(
- Compression::from(parquet::CompressionCodec::Lz4),
- Compression::LZ4
- );
- assert_eq!(
- Compression::from(parquet::CompressionCodec::Zstd),
- Compression::ZSTD
- );
- }
-
- #[test]
- fn test_into_compression() {
- assert_eq!(
- parquet::CompressionCodec::Uncompressed,
- Compression::UNCOMPRESSED.into()
- );
- assert_eq!(
- parquet::CompressionCodec::Snappy,
- Compression::SNAPPY.into()
- );
- assert_eq!(parquet::CompressionCodec::Gzip, Compression::GZIP.into());
- assert_eq!(parquet::CompressionCodec::Lzo, Compression::LZO.into());
- assert_eq!(
- parquet::CompressionCodec::Brotli,
- Compression::BROTLI.into()
- );
- assert_eq!(parquet::CompressionCodec::Lz4, Compression::LZ4.into());
- assert_eq!(parquet::CompressionCodec::Zstd, Compression::ZSTD.into());
- }
-
- #[test]
- fn test_display_page_type() {
- assert_eq!(PageType::DATA_PAGE.to_string(), "DATA_PAGE");
- assert_eq!(PageType::INDEX_PAGE.to_string(), "INDEX_PAGE");
- assert_eq!(PageType::DICTIONARY_PAGE.to_string(), "DICTIONARY_PAGE");
- assert_eq!(PageType::DATA_PAGE_V2.to_string(), "DATA_PAGE_V2");
- }
-
- #[test]
- fn test_from_page_type() {
- assert_eq!(
- PageType::from(parquet::PageType::DataPage),
- PageType::DATA_PAGE
- );
- assert_eq!(
- PageType::from(parquet::PageType::IndexPage),
- PageType::INDEX_PAGE
- );
- assert_eq!(
- PageType::from(parquet::PageType::DictionaryPage),
- PageType::DICTIONARY_PAGE
- );
- assert_eq!(
- PageType::from(parquet::PageType::DataPageV2),
- PageType::DATA_PAGE_V2
- );
- }
-
- #[test]
- fn test_into_page_type() {
- assert_eq!(parquet::PageType::DataPage, PageType::DATA_PAGE.into());
- assert_eq!(parquet::PageType::IndexPage, PageType::INDEX_PAGE.into());
- assert_eq!(
- parquet::PageType::DictionaryPage,
- PageType::DICTIONARY_PAGE.into()
- );
- assert_eq!(parquet::PageType::DataPageV2, PageType::DATA_PAGE_V2.into());
- }
-
- #[test]
- fn test_display_sort_order() {
- assert_eq!(SortOrder::SIGNED.to_string(), "SIGNED");
- assert_eq!(SortOrder::UNSIGNED.to_string(), "UNSIGNED");
- assert_eq!(SortOrder::UNDEFINED.to_string(), "UNDEFINED");
- }
-
- #[test]
- fn test_display_column_order() {
- assert_eq!(
- ColumnOrder::TYPE_DEFINED_ORDER(SortOrder::SIGNED).to_string(),
- "TYPE_DEFINED_ORDER(SIGNED)"
- );
- assert_eq!(
- ColumnOrder::TYPE_DEFINED_ORDER(SortOrder::UNSIGNED).to_string(),
- "TYPE_DEFINED_ORDER(UNSIGNED)"
- );
- assert_eq!(
- ColumnOrder::TYPE_DEFINED_ORDER(SortOrder::UNDEFINED).to_string(),
- "TYPE_DEFINED_ORDER(UNDEFINED)"
- );
- assert_eq!(ColumnOrder::UNDEFINED.to_string(), "UNDEFINED");
- }
-
- #[test]
- fn test_column_order_get_logical_type_sort_order() {
- // Helper to check the order in a list of values.
- // Only logical type is checked.
- fn check_sort_order(types: Vec<LogicalType>, expected_order: SortOrder) {
- for tpe in types {
- assert_eq!(
- ColumnOrder::get_sort_order(
- Some(tpe),
- ConvertedType::NONE,
- Type::BYTE_ARRAY
- ),
- expected_order
- );
- }
- }
-
- // Unsigned comparison (physical type does not matter)
- let unsigned = vec![
- LogicalType::STRING(Default::default()),
- LogicalType::JSON(Default::default()),
- LogicalType::BSON(Default::default()),
- LogicalType::ENUM(Default::default()),
- LogicalType::UUID(Default::default()),
- LogicalType::INTEGER(IntType {
- bit_width: 8,
- is_signed: false,
- }),
- LogicalType::INTEGER(IntType {
- bit_width: 16,
- is_signed: false,
- }),
- LogicalType::INTEGER(IntType {
- bit_width: 32,
- is_signed: false,
- }),
- LogicalType::INTEGER(IntType {
- bit_width: 64,
- is_signed: false,
- }),
- ];
- check_sort_order(unsigned, SortOrder::UNSIGNED);
-
- // Signed comparison (physical type does not matter)
- let signed = vec![
- LogicalType::INTEGER(IntType {
- bit_width: 8,
- is_signed: true,
- }),
- LogicalType::INTEGER(IntType {
- bit_width: 8,
- is_signed: true,
- }),
- LogicalType::INTEGER(IntType {
- bit_width: 8,
- is_signed: true,
- }),
- LogicalType::INTEGER(IntType {
- bit_width: 8,
- is_signed: true,
- }),
- LogicalType::DECIMAL(DecimalType {
- scale: 20,
- precision: 4,
- }),
- LogicalType::DATE(Default::default()),
- LogicalType::TIME(TimeType {
- is_adjusted_to_u_t_c: false,
- unit: TimeUnit::MILLIS(Default::default()),
- }),
- LogicalType::TIME(TimeType {
- is_adjusted_to_u_t_c: false,
- unit: TimeUnit::MICROS(Default::default()),
- }),
- LogicalType::TIME(TimeType {
- is_adjusted_to_u_t_c: true,
- unit: TimeUnit::NANOS(Default::default()),
- }),
- LogicalType::TIMESTAMP(TimestampType {
- is_adjusted_to_u_t_c: false,
- unit: TimeUnit::MILLIS(Default::default()),
- }),
- LogicalType::TIMESTAMP(TimestampType {
- is_adjusted_to_u_t_c: false,
- unit: TimeUnit::MICROS(Default::default()),
- }),
- LogicalType::TIMESTAMP(TimestampType {
- is_adjusted_to_u_t_c: true,
- unit: TimeUnit::NANOS(Default::default()),
- }),
- ];
- check_sort_order(signed, SortOrder::SIGNED);
-
- // Undefined comparison
- let undefined = vec![
- LogicalType::LIST(Default::default()),
- LogicalType::MAP(Default::default()),
- ];
- check_sort_order(undefined, SortOrder::UNDEFINED);
- }
-
- #[test]
- fn test_column_order_get_coverted_type_sort_order() {
- // Helper to check the order in a list of values.
- // Only converted type is checked.
- fn check_sort_order(types: Vec<ConvertedType>, expected_order: SortOrder) {
- for tpe in types {
- assert_eq!(
- ColumnOrder::get_sort_order(None, tpe, Type::BYTE_ARRAY),
- expected_order
- );
- }
- }
-
- // Unsigned comparison (physical type does not matter)
- let unsigned = vec![
- ConvertedType::UTF8,
- ConvertedType::JSON,
- ConvertedType::BSON,
- ConvertedType::ENUM,
- ConvertedType::UINT_8,
- ConvertedType::UINT_16,
- ConvertedType::UINT_32,
- ConvertedType::UINT_64,
- ];
- check_sort_order(unsigned, SortOrder::UNSIGNED);
-
- // Signed comparison (physical type does not matter)
- let signed = vec![
- ConvertedType::INT_8,
- ConvertedType::INT_16,
- ConvertedType::INT_32,
- ConvertedType::INT_64,
- ConvertedType::DECIMAL,
- ConvertedType::DATE,
- ConvertedType::TIME_MILLIS,
- ConvertedType::TIME_MICROS,
- ConvertedType::TIMESTAMP_MILLIS,
- ConvertedType::TIMESTAMP_MICROS,
- ];
- check_sort_order(signed, SortOrder::SIGNED);
-
- // Undefined comparison
- let undefined = vec![
- ConvertedType::LIST,
- ConvertedType::MAP,
- ConvertedType::MAP_KEY_VALUE,
- ConvertedType::INTERVAL,
- ];
- check_sort_order(undefined, SortOrder::UNDEFINED);
-
- // Check None logical type
- // This should return a sort order for byte array type.
- check_sort_order(vec![ConvertedType::NONE], SortOrder::UNSIGNED);
- }
-
- #[test]
- fn test_column_order_get_default_sort_order() {
- // Comparison based on physical type
- assert_eq!(
- ColumnOrder::get_default_sort_order(Type::BOOLEAN),
- SortOrder::UNSIGNED
- );
- assert_eq!(
- ColumnOrder::get_default_sort_order(Type::INT32),
- SortOrder::SIGNED
- );
- assert_eq!(
- ColumnOrder::get_default_sort_order(Type::INT64),
- SortOrder::SIGNED
- );
- assert_eq!(
- ColumnOrder::get_default_sort_order(Type::INT96),
- SortOrder::UNDEFINED
- );
- assert_eq!(
- ColumnOrder::get_default_sort_order(Type::FLOAT),
- SortOrder::SIGNED
- );
- assert_eq!(
- ColumnOrder::get_default_sort_order(Type::DOUBLE),
- SortOrder::SIGNED
- );
- assert_eq!(
- ColumnOrder::get_default_sort_order(Type::BYTE_ARRAY),
- SortOrder::UNSIGNED
- );
- assert_eq!(
- ColumnOrder::get_default_sort_order(Type::FIXED_LEN_BYTE_ARRAY),
- SortOrder::UNSIGNED
- );
- }
-
- #[test]
- fn test_column_order_sort_order() {
- assert_eq!(
- ColumnOrder::TYPE_DEFINED_ORDER(SortOrder::SIGNED).sort_order(),
- SortOrder::SIGNED
- );
- assert_eq!(
- ColumnOrder::TYPE_DEFINED_ORDER(SortOrder::UNSIGNED).sort_order(),
- SortOrder::UNSIGNED
- );
- assert_eq!(
- ColumnOrder::TYPE_DEFINED_ORDER(SortOrder::UNDEFINED).sort_order(),
- SortOrder::UNDEFINED
- );
- assert_eq!(ColumnOrder::UNDEFINED.sort_order(), SortOrder::SIGNED);
- }
-}
diff --git a/parquet/src/bin/parquet-read.rs b/parquet/src/bin/parquet-read.rs
deleted file mode 100644
index aa3b827..0000000
--- a/parquet/src/bin/parquet-read.rs
+++ /dev/null
@@ -1,128 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-//! Binary file to read data from a Parquet file.
-//!
-//! # Install
-//!
-//! `parquet-read` can be installed using `cargo`:
-//! ```
-//! cargo install parquet
-//! ```
-//! After this `parquet-read` should be globally available:
-//! ```
-//! parquet-read XYZ.parquet
-//! ```
-//!
-//! The binary can also be built from the source code and run as follows:
-//! ```
-//! cargo run --bin parquet-read XYZ.parquet
-//! ```
-//!
-//! # Usage
-//! ```
-//! parquet-read <file-path> [num-records]
-//! ```
-//!
-//! ## Flags
-//! -h, --help Prints help information
-//! -j, --json Print Parquet file in JSON lines Format
-//! -V, --version Prints version information
-//!
-//! ## Args
-//! <file-path> Path to a Parquet file
-//! <num-records> Number of records to read. When not provided, all records are read.
-//!
-//! Note that `parquet-read` reads full file schema, no projection or filtering is
-//! applied.
-
-extern crate parquet;
-
-use std::{env, fs::File, path::Path};
-
-use clap::{crate_authors, crate_version, App, Arg};
-
-use parquet::file::reader::{FileReader, SerializedFileReader};
-use parquet::record::Row;
-
-fn main() {
- let app = App::new("parquet-read")
- .version(crate_version!())
- .author(crate_authors!())
- .about("Read data from a Parquet file and print output in console, in either built-in or JSON format")
- .arg(
- Arg::with_name("file_path")
- .value_name("file-path")
- .required(true)
- .index(1)
- .help("Path to a parquet file"),
- )
- .arg(
- Arg::with_name("num_records")
- .value_name("num-records")
- .index(2)
- .help(
- "Number of records to read. When not provided, all records are read.",
- ),
- )
- .arg(
- Arg::with_name("json")
- .short("j")
- .long("json")
- .takes_value(false)
- .help("Print Parquet file in JSON lines format"),
- );
-
- let matches = app.get_matches();
- let filename = matches.value_of("file_path").unwrap();
- let num_records: Option<usize> = if matches.is_present("num_records") {
- match matches.value_of("num_records").unwrap().parse() {
- Ok(value) => Some(value),
- Err(e) => panic!("Error when reading value for [num-records], {}", e),
- }
- } else {
- None
- };
-
- let json = matches.is_present("json");
- let path = Path::new(&filename);
- let file = File::open(&path).unwrap();
- let parquet_reader = SerializedFileReader::new(file).unwrap();
-
- // Use full schema as projected schema
- let mut iter = parquet_reader.get_row_iter(None).unwrap();
-
- let mut start = 0;
- let end = num_records.unwrap_or(0);
- let all_records = num_records.is_none();
-
- while all_records || start < end {
- match iter.next() {
- Some(row) => print_row(&row, json),
- None => break,
- }
- start += 1;
- }
-}
-
-fn print_row(row: &Row, json: bool) {
- if json {
- println!("{}", row.to_json_value())
- } else {
- println!("{}", row.to_string());
- }
-}
diff --git a/parquet/src/bin/parquet-rowcount.rs b/parquet/src/bin/parquet-rowcount.rs
deleted file mode 100644
index 3c61bab..0000000
--- a/parquet/src/bin/parquet-rowcount.rs
+++ /dev/null
@@ -1,87 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-//! Binary file to return the number of rows found from Parquet file(s).
-//!
-//! # Install
-//!
-//! `parquet-rowcount` can be installed using `cargo`:
-//! ```
-//! cargo install parquet
-//! ```
-//! After this `parquet-rowcount` should be globally available:
-//! ```
-//! parquet-rowcount XYZ.parquet
-//! ```
-//!
-//! The binary can also be built from the source code and run as follows:
-//! ```
-//! cargo run --bin parquet-rowcount XYZ.parquet ABC.parquet ZXC.parquet
-//! ```
-//!
-//! # Usage
-//! ```
-//! parquet-rowcount <file-paths>...
-//! ```
-//!
-//! ## Flags
-//! -h, --help Prints help information
-//! -V, --version Prints version information
-//!
-//! ## Args
-//! <file-paths>... List of Parquet files to read from
-//!
-//! Note that `parquet-rowcount` reads full file schema, no projection or filtering is
-//! applied.
-
-extern crate parquet;
-
-use std::{env, fs::File, path::Path};
-
-use clap::{crate_authors, crate_version, App, Arg};
-
-use parquet::file::reader::{FileReader, SerializedFileReader};
-
-fn main() {
- let matches = App::new("parquet-rowcount")
- .version(crate_version!())
- .author(crate_authors!())
- .about("Return number of rows in Parquet file")
- .arg(
- Arg::with_name("file_paths")
- .value_name("file-paths")
- .required(true)
- .multiple(true)
- .help("List of Parquet files to read from separated by space"),
- )
- .get_matches();
-
- let filenames: Vec<&str> = matches.values_of("file_paths").unwrap().collect();
- for filename in &filenames {
- let path = Path::new(filename);
- let file = File::open(path).unwrap();
- let parquet_reader = SerializedFileReader::new(file).unwrap();
- let row_group_metadata = parquet_reader.metadata().row_groups();
- let mut total_num_rows = 0;
-
- for group_metadata in row_group_metadata {
- total_num_rows += group_metadata.num_rows();
- }
-
- eprintln!("File {}: rowcount={}", filename, total_num_rows);
- }
-}
diff --git a/parquet/src/bin/parquet-schema.rs b/parquet/src/bin/parquet-schema.rs
deleted file mode 100644
index 1b80637..0000000
--- a/parquet/src/bin/parquet-schema.rs
+++ /dev/null
@@ -1,104 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-//! Binary file to print the schema and metadata of a Parquet file.
-//!
-//! # Install
-//!
-//! `parquet-schema` can be installed using `cargo`:
-//! ```
-//! cargo install parquet
-//! ```
-//! After this `parquet-schema` should be globally available:
-//! ```
-//! parquet-schema XYZ.parquet
-//! ```
-//!
-//! The binary can also be built from the source code and run as follows:
-//! ```
-//! cargo run --bin parquet-schema XYZ.parquet
-//! ```
-//!
-//! # Usage
-//! ```
-//! parquet-schema [FLAGS] <file-path>
-//! ```
-//!
-//! ## Flags
-//! -h, --help Prints help information
-//! -V, --version Prints version information
-//! -v, --verbose Enable printing full file metadata
-//!
-//! ## Args
-//! <file-path> Path to a Parquet file
-//!
-//! Note that `verbose` is an optional boolean flag that allows to print schema only,
-//! when not provided or print full file metadata when provided.
-
-extern crate parquet;
-
-use std::{env, fs::File, path::Path};
-
-use clap::{crate_authors, crate_version, App, Arg};
-
-use parquet::{
- file::reader::{FileReader, SerializedFileReader},
- schema::printer::{print_file_metadata, print_parquet_metadata},
-};
-
-fn main() {
- let matches = App::new("parquet-schema")
- .version(crate_version!())
- .author(crate_authors!())
- .arg(
- Arg::with_name("file_path")
- .value_name("file-path")
- .required(true)
- .index(1)
- .help("Path to a Parquet file"),
- )
- .arg(
- Arg::with_name("verbose")
- .short("v")
- .long("verbose")
- .takes_value(false)
- .help("Enable printing full file metadata"),
- )
- .get_matches();
-
- let filename = matches.value_of("file_path").unwrap();
- let path = Path::new(&filename);
- let file = match File::open(&path) {
- Err(e) => panic!("Error when opening file {}: {}", path.display(), e),
- Ok(f) => f,
- };
- let verbose = matches.is_present("verbose");
-
- match SerializedFileReader::new(file) {
- Err(e) => panic!("Error when parsing Parquet file: {}", e),
- Ok(parquet_reader) => {
- let metadata = parquet_reader.metadata();
- println!("Metadata for file: {}", &filename);
- println!();
- if verbose {
- print_parquet_metadata(&mut std::io::stdout(), &metadata);
- } else {
- print_file_metadata(&mut std::io::stdout(), &metadata.file_metadata());
- }
- }
- }
-}
diff --git a/parquet/src/column/mod.rs b/parquet/src/column/mod.rs
deleted file mode 100644
index 7ed7bfc..0000000
--- a/parquet/src/column/mod.rs
+++ /dev/null
@@ -1,124 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-//! Low level column reader and writer APIs.
-//!
-//! This API is designed for reading and writing column values, definition and repetition
-//! levels directly.
-//!
-//! # Example of writing and reading data
-//!
-//! Data has the following format:
-//! ```text
-//! +---------------+
-//! | values|
-//! +---------------+
-//! |[1, 2] |
-//! |[3, null, null]|
-//! +---------------+
-//! ```
-//!
-//! The example uses column writer and reader APIs to write raw values, definition and
-//! repetition levels and read them to verify write/read correctness.
-//!
-//! ```rust,no_run
-//! use std::{fs, path::Path, sync::Arc};
-//!
-//! use parquet::{
-//! column::{reader::ColumnReader, writer::ColumnWriter},
-//! file::{
-//! properties::WriterProperties,
-//! reader::{FileReader, SerializedFileReader},
-//! writer::{FileWriter, SerializedFileWriter},
-//! },
-//! schema::parser::parse_message_type,
-//! };
-//!
-//! let path = Path::new("/path/to/column_sample.parquet");
-//!
-//! // Writing data using column writer API.
-//!
-//! let message_type = "
-//! message schema {
-//! optional group values (LIST) {
-//! repeated group list {
-//! optional INT32 element;
-//! }
-//! }
-//! }
-//! ";
-//! let schema = Arc::new(parse_message_type(message_type).unwrap());
-//! let props = Arc::new(WriterProperties::builder().build());
-//! let file = fs::File::create(path).unwrap();
-//! let mut writer = SerializedFileWriter::new(file, schema, props).unwrap();
-//! let mut row_group_writer = writer.next_row_group().unwrap();
-//! while let Some(mut col_writer) = row_group_writer.next_column().unwrap() {
-//! match col_writer {
-//! // You can also use `get_typed_column_writer` method to extract typed writer.
-//! ColumnWriter::Int32ColumnWriter(ref mut typed_writer) => {
-//! typed_writer
-//! .write_batch(&[1, 2, 3], Some(&[3, 3, 3, 2, 2]), Some(&[0, 1, 0, 1, 1]))
-//! .unwrap();
-//! }
-//! _ => {}
-//! }
-//! row_group_writer.close_column(col_writer).unwrap();
-//! }
-//! writer.close_row_group(row_group_writer).unwrap();
-//! writer.close().unwrap();
-//!
-//! // Reading data using column reader API.
-//!
-//! let file = fs::File::open(path).unwrap();
-//! let reader = SerializedFileReader::new(file).unwrap();
-//! let metadata = reader.metadata();
-//!
-//! let mut res = Ok((0, 0));
-//! let mut values = vec![0; 8];
-//! let mut def_levels = vec![0; 8];
-//! let mut rep_levels = vec![0; 8];
-//!
-//! for i in 0..metadata.num_row_groups() {
-//! let row_group_reader = reader.get_row_group(i).unwrap();
-//! let row_group_metadata = metadata.row_group(i);
-//!
-//! for j in 0..row_group_metadata.num_columns() {
-//! let mut column_reader = row_group_reader.get_column_reader(j).unwrap();
-//! match column_reader {
-//! // You can also use `get_typed_column_reader` method to extract typed reader.
-//! ColumnReader::Int32ColumnReader(ref mut typed_reader) => {
-//! res = typed_reader.read_batch(
-//! 8, // batch size
-//! Some(&mut def_levels),
-//! Some(&mut rep_levels),
-//! &mut values,
-//! );
-//! }
-//! _ => {}
-//! }
-//! }
-//! }
-//!
-//! assert_eq!(res, Ok((3, 5)));
-//! assert_eq!(values, vec![1, 2, 3, 0, 0, 0, 0, 0]);
-//! assert_eq!(def_levels, vec![3, 3, 3, 2, 2, 0, 0, 0]);
-//! assert_eq!(rep_levels, vec![0, 1, 0, 1, 1, 0, 0, 0]);
-//! ```
-
-pub mod page;
-pub mod reader;
-pub mod writer;
diff --git a/parquet/src/column/page.rs b/parquet/src/column/page.rs
deleted file mode 100644
index b75d3b5..0000000
--- a/parquet/src/column/page.rs
+++ /dev/null
@@ -1,307 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-//! Contains Parquet Page definitions and page reader interface.
-
-use crate::basic::{Encoding, PageType};
-use crate::errors::Result;
-use crate::file::{metadata::ColumnChunkMetaData, statistics::Statistics};
-use crate::schema::types::{ColumnDescPtr, SchemaDescPtr};
-use crate::util::memory::ByteBufferPtr;
-
-/// Parquet Page definition.
-///
-/// List of supported pages.
-/// These are 1-to-1 mapped from the equivalent Thrift definitions, except `buf` which
-/// used to store uncompressed bytes of the page.
-#[derive(Clone)]
-pub enum Page {
- DataPage {
- buf: ByteBufferPtr,
- num_values: u32,
- encoding: Encoding,
- def_level_encoding: Encoding,
- rep_level_encoding: Encoding,
- statistics: Option<Statistics>,
- },
- DataPageV2 {
- buf: ByteBufferPtr,
- num_values: u32,
- encoding: Encoding,
- num_nulls: u32,
- num_rows: u32,
- def_levels_byte_len: u32,
- rep_levels_byte_len: u32,
- is_compressed: bool,
- statistics: Option<Statistics>,
- },
- DictionaryPage {
- buf: ByteBufferPtr,
- num_values: u32,
- encoding: Encoding,
- is_sorted: bool,
- },
-}
-
-impl Page {
- /// Returns [`PageType`](crate::basic::PageType) for this page.
- pub fn page_type(&self) -> PageType {
- match self {
- Page::DataPage { .. } => PageType::DATA_PAGE,
- Page::DataPageV2 { .. } => PageType::DATA_PAGE_V2,
- Page::DictionaryPage { .. } => PageType::DICTIONARY_PAGE,
- }
- }
-
- /// Returns internal byte buffer reference for this page.
- pub fn buffer(&self) -> &ByteBufferPtr {
- match self {
- Page::DataPage { ref buf, .. } => &buf,
- Page::DataPageV2 { ref buf, .. } => &buf,
- Page::DictionaryPage { ref buf, .. } => &buf,
- }
- }
-
- /// Returns number of values in this page.
- pub fn num_values(&self) -> u32 {
- match self {
- Page::DataPage { num_values, .. } => *num_values,
- Page::DataPageV2 { num_values, .. } => *num_values,
- Page::DictionaryPage { num_values, .. } => *num_values,
- }
- }
-
- /// Returns this page [`Encoding`](crate::basic::Encoding).
- pub fn encoding(&self) -> Encoding {
- match self {
- Page::DataPage { encoding, .. } => *encoding,
- Page::DataPageV2 { encoding, .. } => *encoding,
- Page::DictionaryPage { encoding, .. } => *encoding,
- }
- }
-
- /// Returns optional [`Statistics`](crate::file::statistics::Statistics).
- pub fn statistics(&self) -> Option<&Statistics> {
- match self {
- Page::DataPage { ref statistics, .. } => statistics.as_ref(),
- Page::DataPageV2 { ref statistics, .. } => statistics.as_ref(),
- Page::DictionaryPage { .. } => None,
- }
- }
-}
-
-/// Helper struct to represent pages with potentially compressed buffer (data page v1) or
-/// compressed and concatenated buffer (def levels + rep levels + compressed values for
-/// data page v2).
-///
-/// The difference with `Page` is that `Page` buffer is always uncompressed.
-pub struct CompressedPage {
- compressed_page: Page,
- uncompressed_size: usize,
-}
-
-impl CompressedPage {
- /// Creates `CompressedPage` from a page with potentially compressed buffer and
- /// uncompressed size.
- pub fn new(compressed_page: Page, uncompressed_size: usize) -> Self {
- Self {
- compressed_page,
- uncompressed_size,
- }
- }
-
- /// Returns page type.
- pub fn page_type(&self) -> PageType {
- self.compressed_page.page_type()
- }
-
- /// Returns underlying page with potentially compressed buffer.
- pub fn compressed_page(&self) -> &Page {
- &self.compressed_page
- }
-
- /// Returns uncompressed size in bytes.
- pub fn uncompressed_size(&self) -> usize {
- self.uncompressed_size
- }
-
- /// Returns compressed size in bytes.
- ///
- /// Note that it is assumed that buffer is compressed, but it may not be. In this
- /// case compressed size will be equal to uncompressed size.
- pub fn compressed_size(&self) -> usize {
- self.compressed_page.buffer().len()
- }
-
- /// Number of values in page.
- pub fn num_values(&self) -> u32 {
- self.compressed_page.num_values()
- }
-
- /// Returns encoding for values in page.
- pub fn encoding(&self) -> Encoding {
- self.compressed_page.encoding()
- }
-
- /// Returns slice of compressed buffer in the page.
- pub fn data(&self) -> &[u8] {
- self.compressed_page.buffer().data()
- }
-}
-
-/// Contains page write metrics.
-pub struct PageWriteSpec {
- pub page_type: PageType,
- pub uncompressed_size: usize,
- pub compressed_size: usize,
- pub num_values: u32,
- pub offset: u64,
- pub bytes_written: u64,
-}
-
-impl PageWriteSpec {
- /// Creates new spec with default page write metrics.
- pub fn new() -> Self {
- Self {
- page_type: PageType::DATA_PAGE,
- uncompressed_size: 0,
- compressed_size: 0,
- num_values: 0,
- offset: 0,
- bytes_written: 0,
- }
- }
-}
-
-/// API for reading pages from a column chunk.
-/// This offers a iterator like API to get the next page.
-pub trait PageReader: Iterator<Item = Result<Page>> {
- /// Gets the next page in the column chunk associated with this reader.
- /// Returns `None` if there are no pages left.
- fn get_next_page(&mut self) -> Result<Option<Page>>;
-}
-
-/// API for writing pages in a column chunk.
-///
-/// It is reasonable to assume that all pages will be written in the correct order, e.g.
-/// dictionary page followed by data pages, or a set of data pages, etc.
-pub trait PageWriter {
- /// Writes a page into the output stream/sink.
- /// Returns `PageWriteSpec` that contains information about written page metrics,
- /// including number of bytes, size, number of values, offset, etc.
- ///
- /// This method is called for every compressed page we write into underlying buffer,
- /// either data page or dictionary page.
- fn write_page(&mut self, page: CompressedPage) -> Result<PageWriteSpec>;
-
- /// Writes column chunk metadata into the output stream/sink.
- ///
- /// This method is called once before page writer is closed, normally when writes are
- /// finalised in column writer.
- fn write_metadata(&mut self, metadata: &ColumnChunkMetaData) -> Result<()>;
-
- /// Closes resources and flushes underlying sink.
- /// Page writer should not be used after this method is called.
- fn close(&mut self) -> Result<()>;
-}
-
-/// An iterator over pages of some specific column in a parquet file.
-pub trait PageIterator: Iterator<Item = Result<Box<dyn PageReader>>> {
- /// Get schema of parquet file.
- fn schema(&mut self) -> Result<SchemaDescPtr>;
-
- /// Get column schema of this page iterator.
- fn column_schema(&mut self) -> Result<ColumnDescPtr>;
-}
-
-#[cfg(test)]
-mod tests {
- use super::*;
-
- #[test]
- fn test_page() {
- let data_page = Page::DataPage {
- buf: ByteBufferPtr::new(vec![0, 1, 2]),
- num_values: 10,
- encoding: Encoding::PLAIN,
- def_level_encoding: Encoding::RLE,
- rep_level_encoding: Encoding::RLE,
- statistics: Some(Statistics::int32(Some(1), Some(2), None, 1, true)),
- };
- assert_eq!(data_page.page_type(), PageType::DATA_PAGE);
- assert_eq!(data_page.buffer().data(), vec![0, 1, 2].as_slice());
- assert_eq!(data_page.num_values(), 10);
- assert_eq!(data_page.encoding(), Encoding::PLAIN);
- assert_eq!(
- data_page.statistics(),
- Some(&Statistics::int32(Some(1), Some(2), None, 1, true))
- );
-
- let data_page_v2 = Page::DataPageV2 {
- buf: ByteBufferPtr::new(vec![0, 1, 2]),
- num_values: 10,
- encoding: Encoding::PLAIN,
- num_nulls: 5,
- num_rows: 20,
- def_levels_byte_len: 30,
- rep_levels_byte_len: 40,
- is_compressed: false,
- statistics: Some(Statistics::int32(Some(1), Some(2), None, 1, true)),
- };
- assert_eq!(data_page_v2.page_type(), PageType::DATA_PAGE_V2);
- assert_eq!(data_page_v2.buffer().data(), vec![0, 1, 2].as_slice());
- assert_eq!(data_page_v2.num_values(), 10);
- assert_eq!(data_page_v2.encoding(), Encoding::PLAIN);
- assert_eq!(
- data_page_v2.statistics(),
- Some(&Statistics::int32(Some(1), Some(2), None, 1, true))
- );
-
- let dict_page = Page::DictionaryPage {
- buf: ByteBufferPtr::new(vec![0, 1, 2]),
- num_values: 10,
- encoding: Encoding::PLAIN,
- is_sorted: false,
- };
- assert_eq!(dict_page.page_type(), PageType::DICTIONARY_PAGE);
- assert_eq!(dict_page.buffer().data(), vec![0, 1, 2].as_slice());
- assert_eq!(dict_page.num_values(), 10);
- assert_eq!(dict_page.encoding(), Encoding::PLAIN);
- assert_eq!(dict_page.statistics(), None);
- }
-
- #[test]
- fn test_compressed_page() {
- let data_page = Page::DataPage {
- buf: ByteBufferPtr::new(vec![0, 1, 2]),
- num_values: 10,
- encoding: Encoding::PLAIN,
- def_level_encoding: Encoding::RLE,
- rep_level_encoding: Encoding::RLE,
- statistics: Some(Statistics::int32(Some(1), Some(2), None, 1, true)),
- };
-
- let cpage = CompressedPage::new(data_page, 5);
-
- assert_eq!(cpage.page_type(), PageType::DATA_PAGE);
- assert_eq!(cpage.uncompressed_size(), 5);
- assert_eq!(cpage.compressed_size(), 3);
- assert_eq!(cpage.num_values(), 10);
- assert_eq!(cpage.encoding(), Encoding::PLAIN);
- assert_eq!(cpage.data(), &[0, 1, 2]);
- }
-}
diff --git a/parquet/src/column/reader.rs b/parquet/src/column/reader.rs
deleted file mode 100644
index 63be17b..0000000
--- a/parquet/src/column/reader.rs
+++ /dev/null
@@ -1,1364 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-//! Contains column reader API.
-
-use std::{
- cmp::{max, min},
- collections::HashMap,
-};
-
-use super::page::{Page, PageReader};
-use crate::basic::*;
-use crate::data_type::*;
-use crate::encodings::{
- decoding::{get_decoder, Decoder, DictDecoder, PlainDecoder},
- levels::LevelDecoder,
-};
-use crate::errors::{ParquetError, Result};
-use crate::schema::types::ColumnDescPtr;
-use crate::util::memory::ByteBufferPtr;
-
-/// Column reader for a Parquet type.
-pub enum ColumnReader {
- BoolColumnReader(ColumnReaderImpl<BoolType>),
- Int32ColumnReader(ColumnReaderImpl<Int32Type>),
- Int64ColumnReader(ColumnReaderImpl<Int64Type>),
- Int96ColumnReader(ColumnReaderImpl<Int96Type>),
- FloatColumnReader(ColumnReaderImpl<FloatType>),
- DoubleColumnReader(ColumnReaderImpl<DoubleType>),
- ByteArrayColumnReader(ColumnReaderImpl<ByteArrayType>),
- FixedLenByteArrayColumnReader(ColumnReaderImpl<FixedLenByteArrayType>),
-}
-
-/// Gets a specific column reader corresponding to column descriptor `col_descr`. The
-/// column reader will read from pages in `col_page_reader`.
-pub fn get_column_reader(
- col_descr: ColumnDescPtr,
- col_page_reader: Box<dyn PageReader>,
-) -> ColumnReader {
- match col_descr.physical_type() {
- Type::BOOLEAN => ColumnReader::BoolColumnReader(ColumnReaderImpl::new(
- col_descr,
- col_page_reader,
- )),
- Type::INT32 => ColumnReader::Int32ColumnReader(ColumnReaderImpl::new(
- col_descr,
- col_page_reader,
- )),
- Type::INT64 => ColumnReader::Int64ColumnReader(ColumnReaderImpl::new(
- col_descr,
- col_page_reader,
- )),
- Type::INT96 => ColumnReader::Int96ColumnReader(ColumnReaderImpl::new(
- col_descr,
- col_page_reader,
- )),
- Type::FLOAT => ColumnReader::FloatColumnReader(ColumnReaderImpl::new(
- col_descr,
- col_page_reader,
- )),
- Type::DOUBLE => ColumnReader::DoubleColumnReader(ColumnReaderImpl::new(
- col_descr,
- col_page_reader,
- )),
- Type::BYTE_ARRAY => ColumnReader::ByteArrayColumnReader(ColumnReaderImpl::new(
- col_descr,
- col_page_reader,
- )),
- Type::FIXED_LEN_BYTE_ARRAY => ColumnReader::FixedLenByteArrayColumnReader(
- ColumnReaderImpl::new(col_descr, col_page_reader),
- ),
- }
-}
-
-/// Gets a typed column reader for the specific type `T`, by "up-casting" `col_reader` of
-/// non-generic type to a generic column reader type `ColumnReaderImpl`.
-///
-/// Panics if actual enum value for `col_reader` does not match the type `T`.
-pub fn get_typed_column_reader<T: DataType>(
- col_reader: ColumnReader,
-) -> ColumnReaderImpl<T> {
- T::get_column_reader(col_reader).unwrap_or_else(|| {
- panic!(
- "Failed to convert column reader into a typed column reader for `{}` type",
- T::get_physical_type()
- )
- })
-}
-
-/// Typed value reader for a particular primitive column.
-pub struct ColumnReaderImpl<T: DataType> {
- descr: ColumnDescPtr,
- def_level_decoder: Option<LevelDecoder>,
- rep_level_decoder: Option<LevelDecoder>,
- page_reader: Box<dyn PageReader>,
- current_encoding: Option<Encoding>,
-
- // The total number of values stored in the data page.
- num_buffered_values: u32,
-
- // The number of values from the current data page that has been decoded into memory
- // so far.
- num_decoded_values: u32,
-
- // Cache of decoders for existing encodings
- decoders: HashMap<Encoding, Box<dyn Decoder<T>>>,
-}
-
-impl<T: DataType> ColumnReaderImpl<T> {
- /// Creates new column reader based on column descriptor and page reader.
- pub fn new(descr: ColumnDescPtr, page_reader: Box<dyn PageReader>) -> Self {
- Self {
- descr,
- def_level_decoder: None,
- rep_level_decoder: None,
- page_reader,
- current_encoding: None,
- num_buffered_values: 0,
- num_decoded_values: 0,
- decoders: HashMap::new(),
- }
- }
-
- /// Reads a batch of values of at most `batch_size`.
- ///
- /// This will try to read from the row group, and fills up at most `batch_size` values
- /// for `def_levels`, `rep_levels` and `values`. It will stop either when the row
- /// group is depleted or `batch_size` values has been read, or there is no space
- /// in the input slices (values/definition levels/repetition levels).
- ///
- /// Note that in case the field being read is not required, `values` could contain
- /// less values than `def_levels`. Also note that this will skip reading def / rep
- /// levels if the field is required / not repeated, respectively.
- ///
- /// If `def_levels` or `rep_levels` is `None`, this will also skip reading the
- /// respective levels. This is useful when the caller of this function knows in
- /// advance that the field is required and non-repeated, therefore can avoid
- /// allocating memory for the levels data. Note that if field has definition
- /// levels, but caller provides None, there might be inconsistency between
- /// levels/values (see comments below).
- ///
- /// Returns a tuple where the first element is the actual number of values read,
- /// and the second element is the actual number of levels read.
- #[inline]
- pub fn read_batch(
- &mut self,
- batch_size: usize,
- mut def_levels: Option<&mut [i16]>,
- mut rep_levels: Option<&mut [i16]>,
- values: &mut [T::T],
- ) -> Result<(usize, usize)> {
- let mut values_read = 0;
- let mut levels_read = 0;
-
- // Compute the smallest batch size we can read based on provided slices
- let mut batch_size = min(batch_size, values.len());
- if let Some(ref levels) = def_levels {
- batch_size = min(batch_size, levels.len());
- }
- if let Some(ref levels) = rep_levels {
- batch_size = min(batch_size, levels.len());
- }
-
- // Read exhaustively all pages until we read all batch_size values/levels
- // or there are no more values/levels to read.
- while max(values_read, levels_read) < batch_size {
- if !self.has_next()? {
- break;
- }
-
- // Batch size for the current iteration
- let iter_batch_size = {
- // Compute approximate value based on values decoded so far
- let mut adjusted_size = min(
- batch_size,
- (self.num_buffered_values - self.num_decoded_values) as usize,
- );
-
- // Adjust batch size by taking into account how much data there
- // to read. As batch_size is also smaller than value and level
- // slices (if available), this ensures that available space is not
- // exceeded.
- adjusted_size = min(adjusted_size, batch_size - values_read);
- adjusted_size = min(adjusted_size, batch_size - levels_read);
-
- adjusted_size
- };
-
- let mut values_to_read = 0;
- let mut num_def_levels = 0;
- let mut num_rep_levels = 0;
-
- // If the field is required and non-repeated, there are no definition levels
- if self.descr.max_def_level() > 0 && def_levels.as_ref().is_some() {
- if let Some(ref mut levels) = def_levels {
- num_def_levels = self.read_def_levels(
- &mut levels[levels_read..levels_read + iter_batch_size],
- )?;
- for i in levels_read..levels_read + num_def_levels {
- if levels[i] == self.descr.max_def_level() {
- values_to_read += 1;
- }
- }
- }
- } else {
- // If max definition level == 0, then it is REQUIRED field, read all
- // values. If definition levels are not provided, we still
- // read all values.
- values_to_read = iter_batch_size;
- }
-
- if self.descr.max_rep_level() > 0 && rep_levels.is_some() {
- if let Some(ref mut levels) = rep_levels {
- num_rep_levels = self.read_rep_levels(
- &mut levels[levels_read..levels_read + iter_batch_size],
- )?;
-
- // If definition levels are defined, check that rep levels == def
- // levels
- if def_levels.is_some() {
- assert_eq!(
- num_def_levels, num_rep_levels,
- "Number of decoded rep / def levels did not match"
- );
- }
- }
- }
-
- // At this point we have read values, definition and repetition levels.
- // If both definition and repetition levels are defined, their counts
- // should be equal. Values count is always less or equal to definition levels.
- //
- // Note that if field is not required, but no definition levels are provided,
- // we would read values of batch size and (if provided, of course) repetition
- // levels of batch size - [!] they will not be synced, because only definition
- // levels enforce number of non-null values to read.
-
- let curr_values_read =
- self.read_values(&mut values[values_read..values_read + values_to_read])?;
-
- // Update all "return" counters and internal state.
-
- // This is to account for when def or rep levels are not provided
- let curr_levels_read = max(num_def_levels, num_rep_levels);
- self.num_decoded_values += max(curr_levels_read, curr_values_read) as u32;
- levels_read += curr_levels_read;
- values_read += curr_values_read;
- }
-
- Ok((values_read, levels_read))
- }
-
- /// Reads a new page and set up the decoders for levels, values or dictionary.
- /// Returns false if there's no page left.
- fn read_new_page(&mut self) -> Result<bool> {
- #[allow(while_true)]
- while true {
- match self.page_reader.get_next_page()? {
- // No more page to read
- None => return Ok(false),
- Some(current_page) => {
- match current_page {
- // 1. Dictionary page: configure dictionary for this page.
- p @ Page::DictionaryPage { .. } => {
- self.configure_dictionary(p)?;
- continue;
- }
- // 2. Data page v1
- Page::DataPage {
- buf,
- num_values,
- encoding,
- def_level_encoding,
- rep_level_encoding,
- statistics: _,
- } => {
- self.num_buffered_values = num_values;
- self.num_decoded_values = 0;
-
- let mut buffer_ptr = buf;
-
- if self.descr.max_rep_level() > 0 {
- let mut rep_decoder = LevelDecoder::v1(
- rep_level_encoding,
- self.descr.max_rep_level(),
- );
- let total_bytes = rep_decoder.set_data(
- self.num_buffered_values as usize,
- buffer_ptr.all(),
- );
- buffer_ptr = buffer_ptr.start_from(total_bytes);
- self.rep_level_decoder = Some(rep_decoder);
- }
-
- if self.descr.max_def_level() > 0 {
- let mut def_decoder = LevelDecoder::v1(
- def_level_encoding,
- self.descr.max_def_level(),
- );
- let total_bytes = def_decoder.set_data(
- self.num_buffered_values as usize,
- buffer_ptr.all(),
- );
- buffer_ptr = buffer_ptr.start_from(total_bytes);
- self.def_level_decoder = Some(def_decoder);
- }
-
- // Data page v1 does not have offset, all content of buffer
- // should be passed
- self.set_current_page_encoding(
- encoding,
- &buffer_ptr,
- 0,
- num_values as usize,
- )?;
- return Ok(true);
- }
- // 3. Data page v2
- Page::DataPageV2 {
- buf,
- num_values,
- encoding,
- num_nulls: _,
- num_rows: _,
- def_levels_byte_len,
- rep_levels_byte_len,
- is_compressed: _,
- statistics: _,
- } => {
- self.num_buffered_values = num_values;
- self.num_decoded_values = 0;
-
- let mut offset = 0;
-
- // DataPage v2 only supports RLE encoding for repetition
- // levels
- if self.descr.max_rep_level() > 0 {
- let mut rep_decoder =
- LevelDecoder::v2(self.descr.max_rep_level());
- let bytes_read = rep_decoder.set_data_range(
- self.num_buffered_values as usize,
- &buf,
- offset,
- rep_levels_byte_len as usize,
- );
- offset += bytes_read;
- self.rep_level_decoder = Some(rep_decoder);
- }
-
- // DataPage v2 only supports RLE encoding for definition
- // levels
- if self.descr.max_def_level() > 0 {
- let mut def_decoder =
- LevelDecoder::v2(self.descr.max_def_level());
- let bytes_read = def_decoder.set_data_range(
- self.num_buffered_values as usize,
- &buf,
- offset,
- def_levels_byte_len as usize,
- );
- offset += bytes_read;
- self.def_level_decoder = Some(def_decoder);
- }
-
- self.set_current_page_encoding(
- encoding,
- &buf,
- offset,
- num_values as usize,
- )?;
- return Ok(true);
- }
- };
- }
- }
- }
-
- Ok(true)
- }
-
- /// Resolves and updates encoding and set decoder for the current page
- fn set_current_page_encoding(
- &mut self,
- mut encoding: Encoding,
- buffer_ptr: &ByteBufferPtr,
- offset: usize,
- len: usize,
- ) -> Result<()> {
- if encoding == Encoding::PLAIN_DICTIONARY {
- encoding = Encoding::RLE_DICTIONARY;
- }
-
- let decoder = if encoding == Encoding::RLE_DICTIONARY {
- self.decoders
- .get_mut(&encoding)
- .expect("Decoder for dict should have been set")
- } else {
- // Search cache for data page decoder
- #[allow(clippy::map_entry)]
- if !self.decoders.contains_key(&encoding) {
- // Initialize decoder for this page
- let data_decoder = get_decoder::<T>(self.descr.clone(), encoding)?;
- self.decoders.insert(encoding, data_decoder);
- }
- self.decoders.get_mut(&encoding).unwrap()
- };
-
- decoder.set_data(buffer_ptr.start_from(offset), len as usize)?;
- self.current_encoding = Some(encoding);
- Ok(())
- }
-
- #[inline]
- fn has_next(&mut self) -> Result<bool> {
- if self.num_buffered_values == 0
- || self.num_buffered_values == self.num_decoded_values
- {
- // TODO: should we return false if read_new_page() = true and
- // num_buffered_values = 0?
- if !self.read_new_page()? {
- Ok(false)
- } else {
- Ok(self.num_buffered_values != 0)
- }
- } else {
- Ok(true)
- }
- }
-
- #[inline]
- fn read_rep_levels(&mut self, buffer: &mut [i16]) -> Result<usize> {
- let level_decoder = self
- .rep_level_decoder
- .as_mut()
- .expect("rep_level_decoder be set");
- level_decoder.get(buffer)
- }
-
- #[inline]
- fn read_def_levels(&mut self, buffer: &mut [i16]) -> Result<usize> {
- let level_decoder = self
- .def_level_decoder
- .as_mut()
- .expect("def_level_decoder be set");
- level_decoder.get(buffer)
- }
-
- #[inline]
- fn read_values(&mut self, buffer: &mut [T::T]) -> Result<usize> {
- let encoding = self
- .current_encoding
- .expect("current_encoding should be set");
- let current_decoder = self
- .decoders
- .get_mut(&encoding)
- .unwrap_or_else(|| panic!("decoder for encoding {} should be set", encoding));
- current_decoder.get(buffer)
- }
-
- #[inline]
- fn configure_dictionary(&mut self, page: Page) -> Result<bool> {
- let mut encoding = page.encoding();
- if encoding == Encoding::PLAIN || encoding == Encoding::PLAIN_DICTIONARY {
- encoding = Encoding::RLE_DICTIONARY
- }
-
- if self.decoders.contains_key(&encoding) {
- return Err(general_err!("Column cannot have more than one dictionary"));
- }
-
- if encoding == Encoding::RLE_DICTIONARY {
- let mut dictionary = PlainDecoder::<T>::new(self.descr.type_length());
- let num_values = page.num_values();
- dictionary.set_data(page.buffer().clone(), num_values as usize)?;
-
- let mut decoder = DictDecoder::new();
- decoder.set_dict(Box::new(dictionary))?;
- self.decoders.insert(encoding, Box::new(decoder));
- Ok(true)
- } else {
- Err(nyi_err!(
- "Invalid/Unsupported encoding type for dictionary: {}",
- encoding
- ))
- }
- }
-}
-
-#[cfg(test)]
-mod tests {
- use super::*;
-
- use rand::distributions::uniform::SampleUniform;
- use std::{collections::VecDeque, sync::Arc, vec::IntoIter};
-
- use crate::basic::Type as PhysicalType;
- use crate::column::page::Page;
- use crate::schema::types::{ColumnDescriptor, ColumnPath, Type as SchemaType};
- use crate::util::test_common::make_pages;
-
- const NUM_LEVELS: usize = 128;
- const NUM_PAGES: usize = 2;
- const MAX_DEF_LEVEL: i16 = 5;
- const MAX_REP_LEVEL: i16 = 5;
-
- // Macro to generate test cases
- macro_rules! test {
- // branch for generating i32 cases
- ($test_func:ident, i32, $func:ident, $def_level:expr, $rep_level:expr,
- $num_pages:expr, $num_levels:expr, $batch_size:expr, $min:expr, $max:expr) => {
- test_internal!(
- $test_func,
- Int32Type,
- get_test_int32_type,
- $func,
- $def_level,
- $rep_level,
- $num_pages,
- $num_levels,
- $batch_size,
- $min,
- $max
- );
- };
- // branch for generating i64 cases
- ($test_func:ident, i64, $func:ident, $def_level:expr, $rep_level:expr,
- $num_pages:expr, $num_levels:expr, $batch_size:expr, $min:expr, $max:expr) => {
- test_internal!(
- $test_func,
- Int64Type,
- get_test_int64_type,
- $func,
- $def_level,
- $rep_level,
- $num_pages,
- $num_levels,
- $batch_size,
- $min,
- $max
- );
- };
- }
-
- macro_rules! test_internal {
- ($test_func:ident, $ty:ident, $pty:ident, $func:ident, $def_level:expr,
- $rep_level:expr, $num_pages:expr, $num_levels:expr, $batch_size:expr,
- $min:expr, $max:expr) => {
- #[test]
- fn $test_func() {
- let desc = Arc::new(ColumnDescriptor::new(
- Arc::new($pty()),
- $def_level,
- $rep_level,
- ColumnPath::new(Vec::new()),
- ));
- let mut tester = ColumnReaderTester::<$ty>::new();
- tester.$func(desc, $num_pages, $num_levels, $batch_size, $min, $max);
- }
- };
- }
-
- test!(
- test_read_plain_v1_int32,
- i32,
- plain_v1,
- MAX_DEF_LEVEL,
- MAX_REP_LEVEL,
- NUM_PAGES,
- NUM_LEVELS,
- 16,
- std::i32::MIN,
- std::i32::MAX
- );
- test!(
- test_read_plain_v2_int32,
- i32,
- plain_v2,
- MAX_DEF_LEVEL,
- MAX_REP_LEVEL,
- NUM_PAGES,
- NUM_LEVELS,
- 16,
- std::i32::MIN,
- std::i32::MAX
- );
-
- test!(
- test_read_plain_v1_int32_uneven,
- i32,
- plain_v1,
- MAX_DEF_LEVEL,
- MAX_REP_LEVEL,
- NUM_PAGES,
- NUM_LEVELS,
- 17,
- std::i32::MIN,
- std::i32::MAX
- );
- test!(
- test_read_plain_v2_int32_uneven,
- i32,
- plain_v2,
- MAX_DEF_LEVEL,
- MAX_REP_LEVEL,
- NUM_PAGES,
- NUM_LEVELS,
- 17,
- std::i32::MIN,
- std::i32::MAX
- );
-
- test!(
- test_read_plain_v1_int32_multi_page,
- i32,
- plain_v1,
- MAX_DEF_LEVEL,
- MAX_REP_LEVEL,
- NUM_PAGES,
- NUM_LEVELS,
- 512,
- std::i32::MIN,
- std::i32::MAX
- );
- test!(
- test_read_plain_v2_int32_multi_page,
- i32,
- plain_v2,
- MAX_DEF_LEVEL,
- MAX_REP_LEVEL,
- NUM_PAGES,
- NUM_LEVELS,
- 512,
- std::i32::MIN,
- std::i32::MAX
- );
-
- // test cases when column descriptor has MAX_DEF_LEVEL = 0 and MAX_REP_LEVEL = 0
- test!(
- test_read_plain_v1_int32_required_non_repeated,
- i32,
- plain_v1,
- 0,
- 0,
- NUM_PAGES,
- NUM_LEVELS,
- 16,
- std::i32::MIN,
- std::i32::MAX
- );
- test!(
- test_read_plain_v2_int32_required_non_repeated,
- i32,
- plain_v2,
- 0,
- 0,
- NUM_PAGES,
- NUM_LEVELS,
- 16,
- std::i32::MIN,
- std::i32::MAX
- );
-
- test!(
- test_read_plain_v1_int64,
- i64,
- plain_v1,
- 1,
- 1,
- NUM_PAGES,
- NUM_LEVELS,
- 16,
- std::i64::MIN,
- std::i64::MAX
- );
- test!(
- test_read_plain_v2_int64,
- i64,
- plain_v2,
- 1,
- 1,
- NUM_PAGES,
- NUM_LEVELS,
- 16,
- std::i64::MIN,
- std::i64::MAX
- );
-
- test!(
- test_read_plain_v1_int64_uneven,
- i64,
- plain_v1,
- 1,
- 1,
- NUM_PAGES,
- NUM_LEVELS,
- 17,
- std::i64::MIN,
- std::i64::MAX
- );
- test!(
- test_read_plain_v2_int64_uneven,
- i64,
- plain_v2,
- 1,
- 1,
- NUM_PAGES,
- NUM_LEVELS,
- 17,
- std::i64::MIN,
- std::i64::MAX
- );
-
- test!(
- test_read_plain_v1_int64_multi_page,
- i64,
- plain_v1,
- 1,
- 1,
- NUM_PAGES,
- NUM_LEVELS,
- 512,
- std::i64::MIN,
- std::i64::MAX
- );
- test!(
- test_read_plain_v2_int64_multi_page,
- i64,
- plain_v2,
- 1,
- 1,
- NUM_PAGES,
- NUM_LEVELS,
- 512,
- std::i64::MIN,
- std::i64::MAX
- );
-
- // test cases when column descriptor has MAX_DEF_LEVEL = 0 and MAX_REP_LEVEL = 0
- test!(
- test_read_plain_v1_int64_required_non_repeated,
- i64,
- plain_v1,
- 0,
- 0,
- NUM_PAGES,
- NUM_LEVELS,
- 16,
- std::i64::MIN,
- std::i64::MAX
- );
- test!(
- test_read_plain_v2_int64_required_non_repeated,
- i64,
- plain_v2,
- 0,
- 0,
- NUM_PAGES,
- NUM_LEVELS,
- 16,
- std::i64::MIN,
- std::i64::MAX
- );
-
- test!(
- test_read_dict_v1_int32_small,
- i32,
- dict_v1,
- MAX_DEF_LEVEL,
- MAX_REP_LEVEL,
- 2,
- 2,
- 16,
- 0,
- 3
- );
- test!(
- test_read_dict_v2_int32_small,
- i32,
- dict_v2,
- MAX_DEF_LEVEL,
- MAX_REP_LEVEL,
- 2,
- 2,
- 16,
- 0,
- 3
- );
-
- test!(
- test_read_dict_v1_int32,
- i32,
- dict_v1,
- MAX_DEF_LEVEL,
- MAX_REP_LEVEL,
- NUM_PAGES,
- NUM_LEVELS,
- 16,
- 0,
- 3
- );
- test!(
- test_read_dict_v2_int32,
- i32,
- dict_v2,
- MAX_DEF_LEVEL,
- MAX_REP_LEVEL,
- NUM_PAGES,
- NUM_LEVELS,
- 16,
- 0,
- 3
- );
-
- test!(
- test_read_dict_v1_int32_uneven,
- i32,
- dict_v1,
- MAX_DEF_LEVEL,
- MAX_REP_LEVEL,
- NUM_PAGES,
- NUM_LEVELS,
- 17,
- 0,
- 3
- );
- test!(
- test_read_dict_v2_int32_uneven,
- i32,
- dict_v2,
- MAX_DEF_LEVEL,
- MAX_REP_LEVEL,
- NUM_PAGES,
- NUM_LEVELS,
- 17,
- 0,
- 3
- );
-
- test!(
- test_read_dict_v1_int32_multi_page,
- i32,
- dict_v1,
- MAX_DEF_LEVEL,
- MAX_REP_LEVEL,
- NUM_PAGES,
- NUM_LEVELS,
- 512,
- 0,
- 3
- );
- test!(
- test_read_dict_v2_int32_multi_page,
- i32,
- dict_v2,
- MAX_DEF_LEVEL,
- MAX_REP_LEVEL,
- NUM_PAGES,
- NUM_LEVELS,
- 512,
- 0,
- 3
- );
-
- test!(
- test_read_dict_v1_int64,
- i64,
- dict_v1,
- MAX_DEF_LEVEL,
- MAX_REP_LEVEL,
- NUM_PAGES,
- NUM_LEVELS,
- 16,
- 0,
- 3
- );
- test!(
- test_read_dict_v2_int64,
- i64,
- dict_v2,
- MAX_DEF_LEVEL,
- MAX_REP_LEVEL,
- NUM_PAGES,
- NUM_LEVELS,
- 16,
- 0,
- 3
- );
-
- #[test]
- fn test_read_batch_values_only() {
- test_read_batch_int32(16, &mut [0; 10], None, None); // < batch_size
- test_read_batch_int32(16, &mut [0; 16], None, None); // == batch_size
- test_read_batch_int32(16, &mut [0; 51], None, None); // > batch_size
- }
-
- #[test]
- fn test_read_batch_values_def_levels() {
- test_read_batch_int32(16, &mut [0; 10], Some(&mut [0; 10]), None);
- test_read_batch_int32(16, &mut [0; 16], Some(&mut [0; 16]), None);
- test_read_batch_int32(16, &mut [0; 51], Some(&mut [0; 51]), None);
- }
-
- #[test]
- fn test_read_batch_values_rep_levels() {
- test_read_batch_int32(16, &mut [0; 10], None, Some(&mut [0; 10]));
- test_read_batch_int32(16, &mut [0; 16], None, Some(&mut [0; 16]));
- test_read_batch_int32(16, &mut [0; 51], None, Some(&mut [0; 51]));
- }
-
- #[test]
- fn test_read_batch_different_buf_sizes() {
- test_read_batch_int32(16, &mut [0; 8], Some(&mut [0; 9]), Some(&mut [0; 7]));
- test_read_batch_int32(16, &mut [0; 1], Some(&mut [0; 9]), Some(&mut [0; 3]));
- }
-
- #[test]
- fn test_read_batch_values_def_rep_levels() {
- test_read_batch_int32(
- 128,
- &mut [0; 128],
- Some(&mut [0; 128]),
- Some(&mut [0; 128]),
- );
- }
-
- #[test]
- fn test_read_batch_adjust_after_buffering_page() {
- // This test covers scenario when buffering new page results in setting number
- // of decoded values to 0, resulting on reading `batch_size` of values, but it is
- // larger than we can insert into slice (affects values and levels).
- //
- // Note: values are chosen to reproduce the issue.
- //
- let primitive_type = get_test_int32_type();
- let desc = Arc::new(ColumnDescriptor::new(
- Arc::new(primitive_type),
- 1,
- 1,
- ColumnPath::new(Vec::new()),
- ));
-
- let num_pages = 2;
- let num_levels = 4;
- let batch_size = 5;
- let values = &mut vec![0; 7];
- let def_levels = &mut vec![0; 7];
- let rep_levels = &mut vec![0; 7];
-
- let mut tester = ColumnReaderTester::<Int32Type>::new();
- tester.test_read_batch(
- desc,
- Encoding::RLE_DICTIONARY,
- num_pages,
- num_levels,
- batch_size,
- std::i32::MIN,
- std::i32::MAX,
- values,
- Some(def_levels),
- Some(rep_levels),
- false,
- );
- }
-
- // ----------------------------------------------------------------------
- // Helper methods to make pages and test
- //
- // # Overview
- //
- // Most of the test functionality is implemented in `ColumnReaderTester`, which
- // provides some general data page test methods:
- // - `test_read_batch_general`
- // - `test_read_batch`
- //
- // There are also some high level wrappers that are part of `ColumnReaderTester`:
- // - `plain_v1` -> call `test_read_batch_general` with data page v1 and plain encoding
- // - `plain_v2` -> call `test_read_batch_general` with data page v2 and plain encoding
- // - `dict_v1` -> call `test_read_batch_general` with data page v1 + dictionary page
- // - `dict_v2` -> call `test_read_batch_general` with data page v2 + dictionary page
- //
- // And even higher level wrappers that simplify testing of almost the same test cases:
- // - `get_test_int32_type`, provides dummy schema type
- // - `get_test_int64_type`, provides dummy schema type
- // - `test_read_batch_int32`, wrapper for `read_batch` tests, since they are basically
- // the same, just different def/rep levels and batch size.
- //
- // # Page assembly
- //
- // Page construction and generation of values, definition and repetition levels
- // happens in `make_pages` function.
- // All values are randomly generated based on provided min/max, levels are calculated
- // based on provided max level for column descriptor (which is basically either int32
- // or int64 type in tests) and `levels_per_page` variable.
- //
- // We use `DataPageBuilder` and its implementation `DataPageBuilderImpl` to actually
- // turn values, definition and repetition levels into data pages (either v1 or v2).
- //
- // Those data pages are then stored as part of `TestPageReader` (we just pass vector
- // of generated pages directly), which implements `PageReader` interface.
- //
- // # Comparison
- //
- // This allows us to pass test page reader into column reader, so we can test
- // functionality of column reader - see `test_read_batch`, where we create column
- // reader -> typed column reader, buffer values in `read_batch` method and compare
- // output with generated data.
-
- // Returns dummy Parquet `Type` for primitive field, because most of our tests use
- // INT32 physical type.
- fn get_test_int32_type() -> SchemaType {
- SchemaType::primitive_type_builder("a", PhysicalType::INT32)
- .with_repetition(Repetition::REQUIRED)
- .with_converted_type(ConvertedType::INT_32)
- .with_length(-1)
- .build()
- .expect("build() should be OK")
- }
-
- // Returns dummy Parquet `Type` for INT64 physical type.
- fn get_test_int64_type() -> SchemaType {
- SchemaType::primitive_type_builder("a", PhysicalType::INT64)
- .with_repetition(Repetition::REQUIRED)
- .with_converted_type(ConvertedType::INT_64)
- .with_length(-1)
- .build()
- .expect("build() should be OK")
- }
-
- // Tests `read_batch()` functionality for INT32.
- //
- // This is a high level wrapper on `ColumnReaderTester` that allows us to specify some
- // boilerplate code for setting up definition/repetition levels and column descriptor.
- fn test_read_batch_int32(
- batch_size: usize,
- values: &mut [i32],
- def_levels: Option<&mut [i16]>,
- rep_levels: Option<&mut [i16]>,
- ) {
- let primitive_type = get_test_int32_type();
- // make field is required based on provided slices of levels
- let max_def_level = if def_levels.is_some() {
- MAX_DEF_LEVEL
- } else {
- 0
- };
- let max_rep_level = if def_levels.is_some() {
- MAX_REP_LEVEL
- } else {
- 0
- };
-
- let desc = Arc::new(ColumnDescriptor::new(
- Arc::new(primitive_type),
- max_def_level,
- max_rep_level,
- ColumnPath::new(Vec::new()),
- ));
- let mut tester = ColumnReaderTester::<Int32Type>::new();
- tester.test_read_batch(
- desc,
- Encoding::RLE_DICTIONARY,
- NUM_PAGES,
- NUM_LEVELS,
- batch_size,
- std::i32::MIN,
- std::i32::MAX,
- values,
- def_levels,
- rep_levels,
- false,
- );
- }
-
- struct ColumnReaderTester<T: DataType>
- where
- T::T: PartialOrd + SampleUniform + Copy,
- {
- rep_levels: Vec<i16>,
- def_levels: Vec<i16>,
- values: Vec<T::T>,
- }
-
- impl<T: DataType> ColumnReaderTester<T>
- where
- T::T: PartialOrd + SampleUniform + Copy,
- {
- pub fn new() -> Self {
- Self {
- rep_levels: Vec::new(),
- def_levels: Vec::new(),
- values: Vec::new(),
- }
- }
-
- // Method to generate and test data pages v1
- fn plain_v1(
- &mut self,
- desc: ColumnDescPtr,
- num_pages: usize,
- num_levels: usize,
- batch_size: usize,
- min: T::T,
- max: T::T,
- ) {
- self.test_read_batch_general(
- desc,
- Encoding::PLAIN,
- num_pages,
- num_levels,
- batch_size,
- min,
- max,
- false,
- );
- }
-
- // Method to generate and test data pages v2
- fn plain_v2(
- &mut self,
- desc: ColumnDescPtr,
- num_pages: usize,
- num_levels: usize,
- batch_size: usize,
- min: T::T,
- max: T::T,
- ) {
- self.test_read_batch_general(
- desc,
- Encoding::PLAIN,
- num_pages,
- num_levels,
- batch_size,
- min,
- max,
- true,
- );
- }
-
- // Method to generate and test dictionary page + data pages v1
- fn dict_v1(
- &mut self,
- desc: ColumnDescPtr,
- num_pages: usize,
- num_levels: usize,
- batch_size: usize,
- min: T::T,
- max: T::T,
- ) {
- self.test_read_batch_general(
- desc,
- Encoding::RLE_DICTIONARY,
- num_pages,
- num_levels,
- batch_size,
- min,
- max,
- false,
- );
- }
-
- // Method to generate and test dictionary page + data pages v2
- fn dict_v2(
- &mut self,
- desc: ColumnDescPtr,
- num_pages: usize,
- num_levels: usize,
- batch_size: usize,
- min: T::T,
- max: T::T,
- ) {
- self.test_read_batch_general(
- desc,
- Encoding::RLE_DICTIONARY,
- num_pages,
- num_levels,
- batch_size,
- min,
- max,
- true,
- );
- }
-
- // Helper function for the general case of `read_batch()` where `values`,
- // `def_levels` and `rep_levels` are always provided with enough space.
- fn test_read_batch_general(
- &mut self,
- desc: ColumnDescPtr,
- encoding: Encoding,
- num_pages: usize,
- num_levels: usize,
- batch_size: usize,
- min: T::T,
- max: T::T,
- use_v2: bool,
- ) {
- let mut def_levels = vec![0; num_levels * num_pages];
- let mut rep_levels = vec![0; num_levels * num_pages];
- let mut values = vec![T::T::default(); num_levels * num_pages];
- self.test_read_batch(
- desc,
- encoding,
- num_pages,
- num_levels,
- batch_size,
- min,
- max,
- &mut values,
- Some(&mut def_levels),
- Some(&mut rep_levels),
- use_v2,
- );
- }
-
- // Helper function to test `read_batch()` method with custom buffers for values,
- // definition and repetition levels.
- fn test_read_batch(
- &mut self,
- desc: ColumnDescPtr,
- encoding: Encoding,
- num_pages: usize,
- num_levels: usize,
- batch_size: usize,
- min: T::T,
- max: T::T,
- values: &mut [T::T],
- mut def_levels: Option<&mut [i16]>,
- mut rep_levels: Option<&mut [i16]>,
- use_v2: bool,
- ) {
- let mut pages = VecDeque::new();
- make_pages::<T>(
- desc.clone(),
- encoding,
- num_pages,
- num_levels,
- min,
- max,
- &mut self.def_levels,
- &mut self.rep_levels,
- &mut self.values,
- &mut pages,
- use_v2,
- );
- let max_def_level = desc.max_def_level();
- let page_reader = TestPageReader::new(Vec::from(pages));
- let column_reader: ColumnReader =
- get_column_reader(desc, Box::new(page_reader));
- let mut typed_column_reader = get_typed_column_reader::<T>(column_reader);
-
- let mut curr_values_read = 0;
- let mut curr_levels_read = 0;
- let mut done = false;
- while !done {
- let actual_def_levels =
- def_levels.as_mut().map(|vec| &mut vec[curr_levels_read..]);
- let actual_rep_levels =
- rep_levels.as_mut().map(|vec| &mut vec[curr_levels_read..]);
-
- let (values_read, levels_read) = typed_column_reader
- .read_batch(
- batch_size,
- actual_def_levels,
- actual_rep_levels,
- &mut values[curr_values_read..],
- )
- .expect("read_batch() should be OK");
-
- if values_read == 0 && levels_read == 0 {
- done = true;
- }
-
- curr_values_read += values_read;
- curr_levels_read += levels_read;
- }
-
- assert!(
- values.len() >= curr_values_read,
- "values.len() >= values_read"
- );
- assert_eq!(
- &values[0..curr_values_read],
- &self.values[0..curr_values_read],
- "values content doesn't match"
- );
-
- if let Some(ref levels) = def_levels {
- assert!(
- levels.len() >= curr_levels_read,
- "def_levels.len() >= levels_read"
- );
- assert_eq!(
- &levels[0..curr_levels_read],
- &self.def_levels[0..curr_levels_read],
- "definition levels content doesn't match"
- );
- }
-
- if let Some(ref levels) = rep_levels {
- assert!(
- levels.len() >= curr_levels_read,
- "rep_levels.len() >= levels_read"
- );
- assert_eq!(
- &levels[0..curr_levels_read],
- &self.rep_levels[0..curr_levels_read],
- "repetition levels content doesn't match"
- );
- }
-
- if def_levels.is_none() && rep_levels.is_none() {
- assert!(
- curr_levels_read == 0,
- "expected to read 0 levels, found {}",
- curr_levels_read
- );
- } else if def_levels.is_some() && max_def_level > 0 {
- assert!(
- curr_levels_read >= curr_values_read,
- "expected levels read to be greater than values read"
- );
- }
- }
- }
-
- struct TestPageReader {
- pages: IntoIter<Page>,
- }
-
- impl TestPageReader {
- pub fn new(pages: Vec<Page>) -> Self {
- Self {
- pages: pages.into_iter(),
- }
- }
- }
-
- impl PageReader for TestPageReader {
- fn get_next_page(&mut self) -> Result<Option<Page>> {
- Ok(self.pages.next())
- }
- }
-
- impl Iterator for TestPageReader {
- type Item = Result<Page>;
-
- fn next(&mut self) -> Option<Self::Item> {
- self.get_next_page().transpose()
- }
- }
-}
diff --git a/parquet/src/column/writer.rs b/parquet/src/column/writer.rs
deleted file mode 100644
index 910a9ed..0000000
--- a/parquet/src/column/writer.rs
+++ /dev/null
@@ -1,2020 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-//! Contains column writer API.
-use std::{cmp, collections::VecDeque, convert::TryFrom, marker::PhantomData, sync::Arc};
-
-use crate::basic::{Compression, Encoding, LogicalType, PageType, Type};
-use crate::column::page::{CompressedPage, Page, PageWriteSpec, PageWriter};
-use crate::compression::{create_codec, Codec};
-use crate::data_type::private::ParquetValueType;
-use crate::data_type::AsBytes;
-use crate::data_type::*;
-use crate::encodings::{
- encoding::{get_encoder, DictEncoder, Encoder},
- levels::{max_buffer_size, LevelEncoder},
-};
-use crate::errors::{ParquetError, Result};
-use crate::file::statistics::Statistics;
-use crate::file::{
- metadata::ColumnChunkMetaData,
- properties::{WriterProperties, WriterPropertiesPtr, WriterVersion},
-};
-use crate::schema::types::ColumnDescPtr;
-use crate::util::bit_util::FromBytes;
-use crate::util::memory::{ByteBufferPtr, MemTracker};
-
-/// Column writer for a Parquet type.
-pub enum ColumnWriter {
- BoolColumnWriter(ColumnWriterImpl<BoolType>),
- Int32ColumnWriter(ColumnWriterImpl<Int32Type>),
- Int64ColumnWriter(ColumnWriterImpl<Int64Type>),
- Int96ColumnWriter(ColumnWriterImpl<Int96Type>),
- FloatColumnWriter(ColumnWriterImpl<FloatType>),
- DoubleColumnWriter(ColumnWriterImpl<DoubleType>),
- ByteArrayColumnWriter(ColumnWriterImpl<ByteArrayType>),
- FixedLenByteArrayColumnWriter(ColumnWriterImpl<FixedLenByteArrayType>),
-}
-
-pub enum Level {
- Page,
- Column,
-}
-
-macro_rules! gen_stats_section {
- ($physical_ty: ty, $stat_fn: ident, $min: ident, $max: ident, $distinct: ident, $nulls: ident) => {{
- let min = $min.as_ref().and_then(|v| {
- Some(read_num_bytes!(
- $physical_ty,
- v.as_bytes().len(),
- &v.as_bytes()
- ))
- });
- let max = $max.as_ref().and_then(|v| {
- Some(read_num_bytes!(
- $physical_ty,
- v.as_bytes().len(),
- &v.as_bytes()
- ))
- });
- Statistics::$stat_fn(min, max, $distinct, $nulls, false)
- }};
-}
-
-/// Gets a specific column writer corresponding to column descriptor `descr`.
-pub fn get_column_writer(
- descr: ColumnDescPtr,
- props: WriterPropertiesPtr,
- page_writer: Box<dyn PageWriter>,
-) -> ColumnWriter {
- match descr.physical_type() {
- Type::BOOLEAN => ColumnWriter::BoolColumnWriter(ColumnWriterImpl::new(
- descr,
- props,
- page_writer,
- )),
- Type::INT32 => ColumnWriter::Int32ColumnWriter(ColumnWriterImpl::new(
- descr,
- props,
- page_writer,
- )),
- Type::INT64 => ColumnWriter::Int64ColumnWriter(ColumnWriterImpl::new(
- descr,
- props,
- page_writer,
- )),
- Type::INT96 => ColumnWriter::Int96ColumnWriter(ColumnWriterImpl::new(
- descr,
- props,
- page_writer,
- )),
- Type::FLOAT => ColumnWriter::FloatColumnWriter(ColumnWriterImpl::new(
- descr,
- props,
- page_writer,
- )),
- Type::DOUBLE => ColumnWriter::DoubleColumnWriter(ColumnWriterImpl::new(
- descr,
- props,
- page_writer,
- )),
- Type::BYTE_ARRAY => ColumnWriter::ByteArrayColumnWriter(ColumnWriterImpl::new(
- descr,
- props,
- page_writer,
- )),
- Type::FIXED_LEN_BYTE_ARRAY => ColumnWriter::FixedLenByteArrayColumnWriter(
- ColumnWriterImpl::new(descr, props, page_writer),
- ),
- }
-}
-
-/// Gets a typed column writer for the specific type `T`, by "up-casting" `col_writer` of
-/// non-generic type to a generic column writer type `ColumnWriterImpl`.
-///
-/// Panics if actual enum value for `col_writer` does not match the type `T`.
-pub fn get_typed_column_writer<T: DataType>(
- col_writer: ColumnWriter,
-) -> ColumnWriterImpl<T> {
- T::get_column_writer(col_writer).unwrap_or_else(|| {
- panic!(
- "Failed to convert column writer into a typed column writer for `{}` type",
- T::get_physical_type()
- )
- })
-}
-
-/// Similar to `get_typed_column_writer` but returns a reference.
-pub fn get_typed_column_writer_ref<T: DataType>(
- col_writer: &ColumnWriter,
-) -> &ColumnWriterImpl<T> {
- T::get_column_writer_ref(col_writer).unwrap_or_else(|| {
- panic!(
- "Failed to convert column writer into a typed column writer for `{}` type",
- T::get_physical_type()
- )
- })
-}
-
-/// Similar to `get_typed_column_writer` but returns a reference.
-pub fn get_typed_column_writer_mut<T: DataType>(
- col_writer: &mut ColumnWriter,
-) -> &mut ColumnWriterImpl<T> {
- T::get_column_writer_mut(col_writer).unwrap_or_else(|| {
- panic!(
- "Failed to convert column writer into a typed column writer for `{}` type",
- T::get_physical_type()
- )
- })
-}
-
-/// Typed column writer for a primitive column.
-pub struct ColumnWriterImpl<T: DataType> {
- // Column writer properties
- descr: ColumnDescPtr,
- props: WriterPropertiesPtr,
- page_writer: Box<dyn PageWriter>,
- has_dictionary: bool,
- dict_encoder: Option<DictEncoder<T>>,
- encoder: Box<dyn Encoder<T>>,
- codec: Compression,
- compressor: Option<Box<dyn Codec>>,
- // Metrics per page
- num_buffered_values: u32,
- num_buffered_encoded_values: u32,
- num_buffered_rows: u32,
- min_page_value: Option<T::T>,
- max_page_value: Option<T::T>,
- num_page_nulls: u64,
- page_distinct_count: Option<u64>,
- // Metrics per column writer
- total_bytes_written: u64,
- total_rows_written: u64,
- total_uncompressed_size: u64,
- total_compressed_size: u64,
- total_num_values: u64,
- dictionary_page_offset: Option<u64>,
- data_page_offset: Option<u64>,
- min_column_value: Option<T::T>,
- max_column_value: Option<T::T>,
- num_column_nulls: u64,
- column_distinct_count: Option<u64>,
- // Reused buffers
- def_levels_sink: Vec<i16>,
- rep_levels_sink: Vec<i16>,
- data_pages: VecDeque<CompressedPage>,
- _phantom: PhantomData<T>,
-}
-
-impl<T: DataType> ColumnWriterImpl<T> {
- pub fn new(
- descr: ColumnDescPtr,
- props: WriterPropertiesPtr,
- page_writer: Box<dyn PageWriter>,
- ) -> Self {
- let codec = props.compression(descr.path());
- let compressor = create_codec(codec).unwrap();
-
- // Optionally set dictionary encoder.
- let dict_encoder = if props.dictionary_enabled(descr.path())
- && has_dictionary_support(T::get_physical_type(), &props)
- {
- Some(DictEncoder::new(descr.clone(), Arc::new(MemTracker::new())))
- } else {
- None
- };
-
- // Whether or not this column writer has a dictionary encoding.
- let has_dictionary = dict_encoder.is_some();
-
- // Set either main encoder or fallback encoder.
- let fallback_encoder = get_encoder(
- descr.clone(),
- props
- .encoding(descr.path())
- .unwrap_or_else(|| fallback_encoding(T::get_physical_type(), &props)),
- Arc::new(MemTracker::new()),
- )
- .unwrap();
-
- Self {
- descr,
- props,
- page_writer,
- has_dictionary,
- dict_encoder,
- encoder: fallback_encoder,
- codec,
- compressor,
- num_buffered_values: 0,
- num_buffered_encoded_values: 0,
- num_buffered_rows: 0,
- total_bytes_written: 0,
- total_rows_written: 0,
- total_uncompressed_size: 0,
- total_compressed_size: 0,
- total_num_values: 0,
- dictionary_page_offset: None,
- data_page_offset: None,
- def_levels_sink: vec![],
- rep_levels_sink: vec![],
- data_pages: VecDeque::new(),
- min_page_value: None,
- max_page_value: None,
- num_page_nulls: 0,
- page_distinct_count: None,
- min_column_value: None,
- max_column_value: None,
- num_column_nulls: 0,
- column_distinct_count: None,
- _phantom: PhantomData,
- }
- }
-
- fn write_batch_internal(
- &mut self,
- values: &[T::T],
- def_levels: Option<&[i16]>,
- rep_levels: Option<&[i16]>,
- min: &Option<T::T>,
- max: &Option<T::T>,
- null_count: Option<u64>,
- distinct_count: Option<u64>,
- ) -> Result<usize> {
- // We check for DataPage limits only after we have inserted the values. If a user
- // writes a large number of values, the DataPage size can be well above the limit.
- //
- // The purpose of this chunking is to bound this. Even if a user writes large
- // number of values, the chunking will ensure that we add data page at a
- // reasonable pagesize limit.
-
- // TODO: find out why we don't account for size of levels when we estimate page
- // size.
-
- // Find out the minimal length to prevent index out of bound errors.
- let mut min_len = values.len();
- if let Some(levels) = def_levels {
- min_len = cmp::min(min_len, levels.len());
- }
- if let Some(levels) = rep_levels {
- min_len = cmp::min(min_len, levels.len());
- }
-
- // Find out number of batches to process.
- let write_batch_size = self.props.write_batch_size();
- let num_batches = min_len / write_batch_size;
-
- // Process pre-calculated statistics
- match (min, max) {
- (Some(min), Some(max)) => {
- if self
- .min_column_value
- .as_ref()
- .map_or(true, |v| self.compare_greater(v, min))
- {
- self.min_column_value = Some(min.clone());
- }
- if self
- .max_column_value
- .as_ref()
- .map_or(true, |v| self.compare_greater(max, v))
- {
- self.max_column_value = Some(max.clone());
- }
- }
- (None, Some(_)) | (Some(_), None) => {
- panic!("min/max should be both set or both None")
- }
- (None, None) => {}
- }
-
- if let Some(distinct) = distinct_count {
- self.column_distinct_count =
- Some(self.column_distinct_count.unwrap_or(0) + distinct);
- }
-
- if let Some(nulls) = null_count {
- self.num_column_nulls += nulls;
- }
-
- let calculate_page_stats = (min.is_none() || max.is_none())
- && null_count.is_none()
- && distinct_count.is_none();
-
- let mut values_offset = 0;
- let mut levels_offset = 0;
- for _ in 0..num_batches {
- values_offset += self.write_mini_batch(
- &values[values_offset..values_offset + write_batch_size],
- def_levels.map(|lv| &lv[levels_offset..levels_offset + write_batch_size]),
- rep_levels.map(|lv| &lv[levels_offset..levels_offset + write_batch_size]),
- calculate_page_stats,
- )?;
- levels_offset += write_batch_size;
- }
-
- values_offset += self.write_mini_batch(
- &values[values_offset..],
- def_levels.map(|lv| &lv[levels_offset..]),
- rep_levels.map(|lv| &lv[levels_offset..]),
- calculate_page_stats,
- )?;
-
- // Return total number of values processed.
- Ok(values_offset)
- }
-
- /// Writes batch of values, definition levels and repetition levels.
- /// Returns number of values processed (written).
- ///
- /// If definition and repetition levels are provided, we write fully those levels and
- /// select how many values to write (this number will be returned), since number of
- /// actual written values may be smaller than provided values.
- ///
- /// If only values are provided, then all values are written and the length of
- /// of the values buffer is returned.
- ///
- /// Definition and/or repetition levels can be omitted, if values are
- /// non-nullable and/or non-repeated.
- pub fn write_batch(
- &mut self,
- values: &[T::T],
- def_levels: Option<&[i16]>,
- rep_levels: Option<&[i16]>,
- ) -> Result<usize> {
- self.write_batch_internal(
- values, def_levels, rep_levels, &None, &None, None, None,
- )
- }
-
- /// Writer may optionally provide pre-calculated statistics for this batch, in which case we do
- /// not calculate page level statistics as this will defeat the purpose of speeding up the write
- /// process with pre-calculated statistics.
- pub fn write_batch_with_statistics(
- &mut self,
- values: &[T::T],
- def_levels: Option<&[i16]>,
- rep_levels: Option<&[i16]>,
- min: &Option<T::T>,
- max: &Option<T::T>,
- nulls_count: Option<u64>,
- distinct_count: Option<u64>,
- ) -> Result<usize> {
- self.write_batch_internal(
- values,
- def_levels,
- rep_levels,
- min,
- max,
- nulls_count,
- distinct_count,
- )
- }
-
- /// Returns total number of bytes written by this column writer so far.
- /// This value is also returned when column writer is closed.
- pub fn get_total_bytes_written(&self) -> u64 {
- self.total_bytes_written
- }
-
- /// Returns total number of rows written by this column writer so far.
- /// This value is also returned when column writer is closed.
- pub fn get_total_rows_written(&self) -> u64 {
- self.total_rows_written
- }
-
- /// Finalises writes and closes the column writer.
- /// Returns total bytes written, total rows written and column chunk metadata.
- pub fn close(mut self) -> Result<(u64, u64, ColumnChunkMetaData)> {
- if self.dict_encoder.is_some() {
- self.write_dictionary_page()?;
- }
- self.flush_data_pages()?;
- let metadata = self.write_column_metadata()?;
- self.dict_encoder = None;
- self.page_writer.close()?;
-
- Ok((self.total_bytes_written, self.total_rows_written, metadata))
- }
-
- /// Writes mini batch of values, definition and repetition levels.
- /// This allows fine-grained processing of values and maintaining a reasonable
- /// page size.
- fn write_mini_batch(
- &mut self,
- values: &[T::T],
- def_levels: Option<&[i16]>,
- rep_levels: Option<&[i16]>,
- calculate_page_stats: bool,
- ) -> Result<usize> {
- let mut values_to_write = 0;
-
- // Check if number of definition levels is the same as number of repetition
- // levels.
- if let (Some(def), Some(rep)) = (def_levels, rep_levels) {
- if def.len() != rep.len() {
- return Err(general_err!(
- "Inconsistent length of definition and repetition levels: {} != {}",
- def.len(),
- rep.len()
- ));
- }
- }
-
- // Process definition levels and determine how many values to write.
- let num_values = if self.descr.max_def_level() > 0 {
- let levels = def_levels.ok_or_else(|| {
- general_err!(
- "Definition levels are required, because max definition level = {}",
- self.descr.max_def_level()
- )
- })?;
-
- for &level in levels {
- if level == self.descr.max_def_level() {
- values_to_write += 1;
- } else if calculate_page_stats {
- self.num_page_nulls += 1
- }
- }
-
- self.write_definition_levels(levels);
- u32::try_from(levels.len()).unwrap()
- } else {
- values_to_write = values.len();
- u32::try_from(values_to_write).unwrap()
- };
-
- // Process repetition levels and determine how many rows we are about to process.
- if self.descr.max_rep_level() > 0 {
- // A row could contain more than one value.
- let levels = rep_levels.ok_or_else(|| {
- general_err!(
- "Repetition levels are required, because max repetition level = {}",
- self.descr.max_rep_level()
- )
- })?;
-
- // Count the occasions where we start a new row
- for &level in levels {
- self.num_buffered_rows += (level == 0) as u32
- }
-
- self.write_repetition_levels(levels);
- } else {
- // Each value is exactly one row.
- // Equals to the number of values, we count nulls as well.
- self.num_buffered_rows += num_values;
- }
-
- // Check that we have enough values to write.
- let values_to_write = values.get(0..values_to_write).ok_or_else(|| {
- general_err!(
- "Expected to write {} values, but have only {}",
- values_to_write,
- values.len()
- )
- })?;
-
- if calculate_page_stats {
- for val in values_to_write {
- self.update_page_min_max(val);
- }
- }
-
- self.write_values(values_to_write)?;
-
- self.num_buffered_values += num_values;
- self.num_buffered_encoded_values += u32::try_from(values_to_write.len()).unwrap();
-
- if self.should_add_data_page() {
- self.add_data_page(calculate_page_stats)?;
- }
-
- if self.should_dict_fallback() {
- self.dict_fallback()?;
- }
-
- Ok(values_to_write.len())
- }
-
- #[inline]
- fn write_definition_levels(&mut self, def_levels: &[i16]) {
- self.def_levels_sink.extend_from_slice(def_levels);
- }
-
- #[inline]
- fn write_repetition_levels(&mut self, rep_levels: &[i16]) {
- self.rep_levels_sink.extend_from_slice(rep_levels);
- }
-
- #[inline]
- fn write_values(&mut self, values: &[T::T]) -> Result<()> {
- match self.dict_encoder {
- Some(ref mut encoder) => encoder.put(values),
- None => self.encoder.put(values),
- }
- }
-
- /// Returns true if we need to fall back to non-dictionary encoding.
- ///
- /// We can only fall back if dictionary encoder is set and we have exceeded dictionary
- /// size.
- #[inline]
- fn should_dict_fallback(&self) -> bool {
- match self.dict_encoder {
- Some(ref encoder) => {
- encoder.dict_encoded_size() >= self.props.dictionary_pagesize_limit()
- }
- None => false,
- }
- }
-
- /// Returns true if there is enough data for a data page, false otherwise.
- #[inline]
- fn should_add_data_page(&self) -> bool {
- match self.dict_encoder {
- Some(ref encoder) => {
- encoder.estimated_data_encoded_size() >= self.props.data_pagesize_limit()
- }
- None => {
- self.encoder.estimated_data_encoded_size()
- >= self.props.data_pagesize_limit()
- }
- }
- }
-
- /// Performs dictionary fallback.
- /// Prepares and writes dictionary and all data pages into page writer.
- fn dict_fallback(&mut self) -> Result<()> {
- // At this point we know that we need to fall back.
- self.write_dictionary_page()?;
- self.flush_data_pages()?;
- self.dict_encoder = None;
- Ok(())
- }
-
- /// Adds data page.
- /// Data page is either buffered in case of dictionary encoding or written directly.
- fn add_data_page(&mut self, calculate_page_stat: bool) -> Result<()> {
- // Extract encoded values
- let value_bytes = match self.dict_encoder {
- Some(ref mut encoder) => encoder.write_indices()?,
- None => self.encoder.flush_buffer()?,
- };
-
- // Select encoding based on current encoder and writer version (v1 or v2).
- let encoding = if self.dict_encoder.is_some() {
- self.props.dictionary_data_page_encoding()
- } else {
- self.encoder.encoding()
- };
-
- let max_def_level = self.descr.max_def_level();
- let max_rep_level = self.descr.max_rep_level();
-
- // always update column NULL count, no matter if page stats are used
- self.num_column_nulls += self.num_page_nulls;
-
- let page_statistics = if calculate_page_stat {
- self.update_column_min_max();
- Some(self.make_page_statistics())
- } else {
- None
- };
-
- let compressed_page = match self.props.writer_version() {
- WriterVersion::PARQUET_1_0 => {
- let mut buffer = vec![];
-
- if max_rep_level > 0 {
- buffer.extend_from_slice(
- &self.encode_levels_v1(
- Encoding::RLE,
- &self.rep_levels_sink[..],
- max_rep_level,
- )?[..],
- );
- }
-
- if max_def_level > 0 {
- buffer.extend_from_slice(
- &self.encode_levels_v1(
- Encoding::RLE,
- &self.def_levels_sink[..],
- max_def_level,
- )?[..],
- );
- }
-
- buffer.extend_from_slice(value_bytes.data());
- let uncompressed_size = buffer.len();
-
- if let Some(ref mut cmpr) = self.compressor {
- let mut compressed_buf = Vec::with_capacity(value_bytes.data().len());
- cmpr.compress(&buffer[..], &mut compressed_buf)?;
- buffer = compressed_buf;
- }
-
- let data_page = Page::DataPage {
- buf: ByteBufferPtr::new(buffer),
- num_values: self.num_buffered_values,
- encoding,
- def_level_encoding: Encoding::RLE,
- rep_level_encoding: Encoding::RLE,
- statistics: page_statistics,
- };
-
- CompressedPage::new(data_page, uncompressed_size)
- }
- WriterVersion::PARQUET_2_0 => {
- let mut rep_levels_byte_len = 0;
- let mut def_levels_byte_len = 0;
- let mut buffer = vec![];
-
- if max_rep_level > 0 {
- let levels =
- self.encode_levels_v2(&self.rep_levels_sink[..], max_rep_level)?;
- rep_levels_byte_len = levels.len();
- buffer.extend_from_slice(&levels[..]);
- }
-
- if max_def_level > 0 {
- let levels =
- self.encode_levels_v2(&self.def_levels_sink[..], max_def_level)?;
- def_levels_byte_len = levels.len();
- buffer.extend_from_slice(&levels[..]);
- }
-
- let uncompressed_size =
- rep_levels_byte_len + def_levels_byte_len + value_bytes.len();
-
- // Data Page v2 compresses values only.
- match self.compressor {
- Some(ref mut cmpr) => {
- cmpr.compress(value_bytes.data(), &mut buffer)?;
- }
- None => buffer.extend_from_slice(value_bytes.data()),
- }
-
- let data_page = Page::DataPageV2 {
- buf: ByteBufferPtr::new(buffer),
- num_values: self.num_buffered_values,
- encoding,
- num_nulls: self.num_buffered_values
- - self.num_buffered_encoded_values,
- num_rows: self.num_buffered_rows,
- def_levels_byte_len: def_levels_byte_len as u32,
- rep_levels_byte_len: rep_levels_byte_len as u32,
- is_compressed: self.compressor.is_some(),
- statistics: page_statistics,
- };
-
- CompressedPage::new(data_page, uncompressed_size)
- }
- };
-
- // Check if we need to buffer data page or flush it to the sink directly.
- if self.dict_encoder.is_some() {
- self.data_pages.push_back(compressed_page);
- } else {
- self.write_data_page(compressed_page)?;
- }
-
- // Update total number of rows.
- self.total_rows_written += self.num_buffered_rows as u64;
-
- // Reset state.
- self.rep_levels_sink.clear();
- self.def_levels_sink.clear();
- self.num_buffered_values = 0;
- self.num_buffered_encoded_values = 0;
- self.num_buffered_rows = 0;
- self.min_page_value = None;
- self.max_page_value = None;
- self.num_page_nulls = 0;
- self.page_distinct_count = None;
-
- Ok(())
- }
-
- /// Finalises any outstanding data pages and flushes buffered data pages from
- /// dictionary encoding into underlying sink.
- #[inline]
- fn flush_data_pages(&mut self) -> Result<()> {
- // Write all outstanding data to a new page.
- let calculate_page_stats =
- self.min_page_value.is_some() && self.max_page_value.is_some();
- if self.num_buffered_values > 0 {
- self.add_data_page(calculate_page_stats)?;
- }
-
- while let Some(page) = self.data_pages.pop_front() {
- self.write_data_page(page)?;
- }
-
- Ok(())
- }
-
- /// Assembles and writes column chunk metadata.
- fn write_column_metadata(&mut self) -> Result<ColumnChunkMetaData> {
- let total_compressed_size = self.total_compressed_size as i64;
- let total_uncompressed_size = self.total_uncompressed_size as i64;
- let num_values = self.total_num_values as i64;
- let dict_page_offset = self.dictionary_page_offset.map(|v| v as i64);
- // If data page offset is not set, then no pages have been written
- let data_page_offset = self.data_page_offset.unwrap_or(0) as i64;
-
- let file_offset;
- let mut encodings = Vec::new();
-
- if self.has_dictionary {
- assert!(dict_page_offset.is_some(), "Dictionary offset is not set");
- file_offset = dict_page_offset.unwrap() + total_compressed_size;
- // NOTE: This should be in sync with writing dictionary pages.
- encodings.push(self.props.dictionary_page_encoding());
- encodings.push(self.props.dictionary_data_page_encoding());
- // Fallback to alternative encoding, add it to the list.
- if self.dict_encoder.is_none() {
- encodings.push(self.encoder.encoding());
- }
- } else {
- file_offset = data_page_offset + total_compressed_size;
- encodings.push(self.encoder.encoding());
- }
- // We use only RLE level encoding for data page v1 and data page v2.
- encodings.push(Encoding::RLE);
-
- let statistics = self.make_column_statistics();
- let metadata = ColumnChunkMetaData::builder(self.descr.clone())
- .set_compression(self.codec)
- .set_encodings(encodings)
- .set_file_offset(file_offset)
- .set_total_compressed_size(total_compressed_size)
- .set_total_uncompressed_size(total_uncompressed_size)
- .set_num_values(num_values)
- .set_data_page_offset(data_page_offset)
- .set_dictionary_page_offset(dict_page_offset)
- .set_statistics(statistics)
- .build()?;
-
- self.page_writer.write_metadata(&metadata)?;
-
- Ok(metadata)
- }
-
- /// Encodes definition or repetition levels for Data Page v1.
- #[inline]
- fn encode_levels_v1(
- &self,
- encoding: Encoding,
- levels: &[i16],
- max_level: i16,
- ) -> Result<Vec<u8>> {
- let size = max_buffer_size(encoding, max_level, levels.len());
- let mut encoder = LevelEncoder::v1(encoding, max_level, vec![0; size]);
- encoder.put(&levels)?;
- encoder.consume()
- }
-
- /// Encodes definition or repetition levels for Data Page v2.
- /// Encoding is always RLE.
- #[inline]
- fn encode_levels_v2(&self, levels: &[i16], max_level: i16) -> Result<Vec<u8>> {
- let size = max_buffer_size(Encoding::RLE, max_level, levels.len());
- let mut encoder = LevelEncoder::v2(max_level, vec![0; size]);
- encoder.put(&levels)?;
- encoder.consume()
- }
-
- /// Writes compressed data page into underlying sink and updates global metrics.
- #[inline]
- fn write_data_page(&mut self, page: CompressedPage) -> Result<()> {
- let page_spec = self.page_writer.write_page(page)?;
- self.update_metrics_for_page(page_spec);
- Ok(())
- }
-
- /// Writes dictionary page into underlying sink.
- #[inline]
- fn write_dictionary_page(&mut self) -> Result<()> {
- let compressed_page = {
- let encoder = self
- .dict_encoder
- .as_ref()
- .ok_or_else(|| general_err!("Dictionary encoder is not set"))?;
-
- let is_sorted = encoder.is_sorted();
- let num_values = encoder.num_entries();
- let mut values_buf = encoder.write_dict()?;
- let uncompressed_size = values_buf.len();
-
- if let Some(ref mut cmpr) = self.compressor {
- let mut output_buf = Vec::with_capacity(uncompressed_size);
- cmpr.compress(values_buf.data(), &mut output_buf)?;
- values_buf = ByteBufferPtr::new(output_buf);
- }
-
- let dict_page = Page::DictionaryPage {
- buf: values_buf,
- num_values: num_values as u32,
- encoding: self.props.dictionary_page_encoding(),
- is_sorted,
- };
- CompressedPage::new(dict_page, uncompressed_size)
- };
-
- let page_spec = self.page_writer.write_page(compressed_page)?;
- self.update_metrics_for_page(page_spec);
- Ok(())
- }
-
- /// Updates column writer metrics with each page metadata.
- #[inline]
- fn update_metrics_for_page(&mut self, page_spec: PageWriteSpec) {
- self.total_uncompressed_size += page_spec.uncompressed_size as u64;
- self.total_compressed_size += page_spec.compressed_size as u64;
- self.total_num_values += page_spec.num_values as u64;
- self.total_bytes_written += page_spec.bytes_written;
-
- match page_spec.page_type {
- PageType::DATA_PAGE | PageType::DATA_PAGE_V2 => {
- if self.data_page_offset.is_none() {
- self.data_page_offset = Some(page_spec.offset);
- }
- }
- PageType::DICTIONARY_PAGE => {
- assert!(
- self.dictionary_page_offset.is_none(),
- "Dictionary offset is already set"
- );
- self.dictionary_page_offset = Some(page_spec.offset);
- }
- _ => {}
- }
- }
-
- /// Returns reference to the underlying page writer.
- /// This method is intended to use in tests only.
- fn get_page_writer_ref(&self) -> &dyn PageWriter {
- self.page_writer.as_ref()
- }
-
- fn make_column_statistics(&self) -> Statistics {
- self.make_typed_statistics(Level::Column)
- }
-
- fn make_page_statistics(&self) -> Statistics {
- self.make_typed_statistics(Level::Page)
- }
-
- pub fn make_typed_statistics(&self, level: Level) -> Statistics {
- let (min, max, distinct, nulls) = match level {
- Level::Page => (
- self.min_page_value.as_ref(),
- self.max_page_value.as_ref(),
- self.page_distinct_count,
- self.num_page_nulls,
- ),
- Level::Column => (
- self.min_column_value.as_ref(),
- self.max_column_value.as_ref(),
- self.column_distinct_count,
- self.num_column_nulls,
- ),
- };
- match self.descr.physical_type() {
- Type::INT32 => gen_stats_section!(i32, int32, min, max, distinct, nulls),
- Type::BOOLEAN => gen_stats_section!(i32, int32, min, max, distinct, nulls),
- Type::INT64 => gen_stats_section!(i64, int64, min, max, distinct, nulls),
- Type::INT96 => gen_stats_section!(Int96, int96, min, max, distinct, nulls),
- Type::FLOAT => gen_stats_section!(f32, float, min, max, distinct, nulls),
- Type::DOUBLE => gen_stats_section!(f64, double, min, max, distinct, nulls),
- Type::BYTE_ARRAY | Type::FIXED_LEN_BYTE_ARRAY => {
- let min = min.as_ref().map(|v| ByteArray::from(v.as_bytes().to_vec()));
- let max = max.as_ref().map(|v| ByteArray::from(v.as_bytes().to_vec()));
- Statistics::byte_array(min, max, distinct, nulls, false)
- }
- }
- }
-
- #[allow(clippy::eq_op)]
- fn update_page_min_max(&mut self, val: &T::T) {
- // simple "isNaN" check that works for all types
- if val == val {
- if self
- .min_page_value
- .as_ref()
- .map_or(true, |min| self.compare_greater(min, val))
- {
- self.min_page_value = Some(val.clone());
- }
- if self
- .max_page_value
- .as_ref()
- .map_or(true, |max| self.compare_greater(val, max))
- {
- self.max_page_value = Some(val.clone());
- }
- }
- }
-
- fn update_column_min_max(&mut self) {
- let update_min = self.min_column_value.as_ref().map_or(true, |min| {
- let page_value = self.min_page_value.as_ref().unwrap();
- self.compare_greater(min, page_value)
- });
- if update_min {
- self.min_column_value = self.min_page_value.clone();
- }
-
- let update_max = self.max_column_value.as_ref().map_or(true, |max| {
- let page_value = self.max_page_value.as_ref().unwrap();
- self.compare_greater(page_value, max)
- });
- if update_max {
- self.max_column_value = self.max_page_value.clone();
- }
- }
-
- /// Evaluate `a > b` according to underlying logical type.
- fn compare_greater(&self, a: &T::T, b: &T::T) -> bool {
- if let Some(LogicalType::INTEGER(int_type)) = self.descr.logical_type() {
- if !int_type.is_signed {
- // need to compare unsigned
- return a.as_u64().unwrap() > b.as_u64().unwrap();
- }
- }
- a > b
- }
-}
-
-// ----------------------------------------------------------------------
-// Encoding support for column writer.
-// This mirrors parquet-mr default encodings for writes. See:
-// https://github.com/apache/parquet-mr/blob/master/parquet-column/src/main/java/org/apache/parquet/column/values/factory/DefaultV1ValuesWriterFactory.java
-// https://github.com/apache/parquet-mr/blob/master/parquet-column/src/main/java/org/apache/parquet/column/values/factory/DefaultV2ValuesWriterFactory.java
-
-/// Trait to define default encoding for types, including whether or not the type
-/// supports dictionary encoding.
-trait EncodingWriteSupport {
- /// Returns true if dictionary is supported for column writer, false otherwise.
- fn has_dictionary_support(props: &WriterProperties) -> bool;
-}
-
-/// Returns encoding for a column when no other encoding is provided in writer properties.
-fn fallback_encoding(kind: Type, props: &WriterProperties) -> Encoding {
- match (kind, props.writer_version()) {
- (Type::BOOLEAN, WriterVersion::PARQUET_2_0) => Encoding::RLE,
- (Type::INT32, WriterVersion::PARQUET_2_0) => Encoding::DELTA_BINARY_PACKED,
- (Type::INT64, WriterVersion::PARQUET_2_0) => Encoding::DELTA_BINARY_PACKED,
- (Type::BYTE_ARRAY, WriterVersion::PARQUET_2_0) => Encoding::DELTA_BYTE_ARRAY,
- (Type::FIXED_LEN_BYTE_ARRAY, WriterVersion::PARQUET_2_0) => {
- Encoding::DELTA_BYTE_ARRAY
- }
- _ => Encoding::PLAIN,
- }
-}
-
-/// Returns true if dictionary is supported for column writer, false otherwise.
-fn has_dictionary_support(kind: Type, props: &WriterProperties) -> bool {
- match (kind, props.writer_version()) {
- // Booleans do not support dict encoding and should use a fallback encoding.
- (Type::BOOLEAN, _) => false,
- // Dictionary encoding was not enabled in PARQUET 1.0
- (Type::FIXED_LEN_BYTE_ARRAY, WriterVersion::PARQUET_1_0) => false,
- (Type::FIXED_LEN_BYTE_ARRAY, WriterVersion::PARQUET_2_0) => true,
- _ => true,
- }
-}
-
-#[cfg(test)]
-mod tests {
- use rand::distributions::uniform::SampleUniform;
-
- use crate::column::{
- page::PageReader,
- reader::{get_column_reader, get_typed_column_reader, ColumnReaderImpl},
- };
- use crate::file::{
- properties::WriterProperties, reader::SerializedPageReader,
- writer::SerializedPageWriter,
- };
- use crate::schema::types::{ColumnDescriptor, ColumnPath, Type as SchemaType};
- use crate::util::{
- io::{FileSink, FileSource},
- test_common::{get_temp_file, random_numbers_range},
- };
-
- use super::*;
-
- #[test]
- fn test_column_writer_inconsistent_def_rep_length() {
- let page_writer = get_test_page_writer();
- let props = Arc::new(WriterProperties::builder().build());
- let mut writer = get_test_column_writer::<Int32Type>(page_writer, 1, 1, props);
- let res = writer.write_batch(&[1, 2, 3, 4], Some(&[1, 1, 1]), Some(&[0, 0]));
- assert!(res.is_err());
- if let Err(err) = res {
- assert_eq!(
- format!("{}", err),
- "Parquet error: Inconsistent length of definition and repetition levels: 3 != 2"
- );
- }
- }
-
- #[test]
- fn test_column_writer_invalid_def_levels() {
- let page_writer = get_test_page_writer();
- let props = Arc::new(WriterProperties::builder().build());
- let mut writer = get_test_column_writer::<Int32Type>(page_writer, 1, 0, props);
- let res = writer.write_batch(&[1, 2, 3, 4], None, None);
- assert!(res.is_err());
- if let Err(err) = res {
- assert_eq!(
- format!("{}", err),
- "Parquet error: Definition levels are required, because max definition level = 1"
- );
- }
- }
-
- #[test]
- fn test_column_writer_invalid_rep_levels() {
- let page_writer = get_test_page_writer();
- let props = Arc::new(WriterProperties::builder().build());
- let mut writer = get_test_column_writer::<Int32Type>(page_writer, 0, 1, props);
- let res = writer.write_batch(&[1, 2, 3, 4], None, None);
- assert!(res.is_err());
- if let Err(err) = res {
- assert_eq!(
- format!("{}", err),
- "Parquet error: Repetition levels are required, because max repetition level = 1"
- );
- }
- }
-
- #[test]
- fn test_column_writer_not_enough_values_to_write() {
- let page_writer = get_test_page_writer();
- let props = Arc::new(WriterProperties::builder().build());
- let mut writer = get_test_column_writer::<Int32Type>(page_writer, 1, 0, props);
- let res = writer.write_batch(&[1, 2], Some(&[1, 1, 1, 1]), None);
- assert!(res.is_err());
- if let Err(err) = res {
- assert_eq!(
- format!("{}", err),
- "Parquet error: Expected to write 4 values, but have only 2"
- );
- }
- }
-
- #[test]
- #[should_panic(expected = "Dictionary offset is already set")]
- fn test_column_writer_write_only_one_dictionary_page() {
- let page_writer = get_test_page_writer();
- let props = Arc::new(WriterProperties::builder().build());
- let mut writer = get_test_column_writer::<Int32Type>(page_writer, 0, 0, props);
- writer.write_batch(&[1, 2, 3, 4], None, None).unwrap();
- // First page should be correctly written.
- let res = writer.write_dictionary_page();
- assert!(res.is_ok());
- writer.write_dictionary_page().unwrap();
- }
-
- #[test]
- fn test_column_writer_error_when_writing_disabled_dictionary() {
- let page_writer = get_test_page_writer();
- let props = Arc::new(
- WriterProperties::builder()
- .set_dictionary_enabled(false)
- .build(),
- );
- let mut writer = get_test_column_writer::<Int32Type>(page_writer, 0, 0, props);
- writer.write_batch(&[1, 2, 3, 4], None, None).unwrap();
- let res = writer.write_dictionary_page();
- assert!(res.is_err());
- if let Err(err) = res {
- assert_eq!(
- format!("{}", err),
- "Parquet error: Dictionary encoder is not set"
- );
- }
- }
-
- #[test]
- fn test_column_writer_boolean_type_does_not_support_dictionary() {
- let page_writer = get_test_page_writer();
- let props = Arc::new(
- WriterProperties::builder()
- .set_dictionary_enabled(true)
- .build(),
- );
- let mut writer = get_test_column_writer::<BoolType>(page_writer, 0, 0, props);
- writer
- .write_batch(&[true, false, true, false], None, None)
- .unwrap();
-
- let (bytes_written, rows_written, metadata) = writer.close().unwrap();
- // PlainEncoder uses bit writer to write boolean values, which all fit into 1
- // byte.
- assert_eq!(bytes_written, 1);
- assert_eq!(rows_written, 4);
- assert_eq!(metadata.encodings(), &vec![Encoding::PLAIN, Encoding::RLE]);
- assert_eq!(metadata.num_values(), 4); // just values
- assert_eq!(metadata.dictionary_page_offset(), None);
- }
-
- #[test]
- fn test_column_writer_default_encoding_support_bool() {
- check_encoding_write_support::<BoolType>(
- WriterVersion::PARQUET_1_0,
- true,
- &[true, false],
- None,
- &[Encoding::PLAIN, Encoding::RLE],
- );
- check_encoding_write_support::<BoolType>(
- WriterVersion::PARQUET_1_0,
- false,
- &[true, false],
- None,
- &[Encoding::PLAIN, Encoding::RLE],
- );
- check_encoding_write_support::<BoolType>(
- WriterVersion::PARQUET_2_0,
- true,
- &[true, false],
- None,
- &[Encoding::RLE, Encoding::RLE],
- );
- check_encoding_write_support::<BoolType>(
- WriterVersion::PARQUET_2_0,
- false,
- &[true, false],
- None,
- &[Encoding::RLE, Encoding::RLE],
- );
- }
-
- #[test]
- fn test_column_writer_default_encoding_support_int32() {
- check_encoding_write_support::<Int32Type>(
- WriterVersion::PARQUET_1_0,
- true,
- &[1, 2],
- Some(0),
- &[Encoding::PLAIN, Encoding::RLE_DICTIONARY, Encoding::RLE],
- );
- check_encoding_write_support::<Int32Type>(
- WriterVersion::PARQUET_1_0,
- false,
- &[1, 2],
- None,
- &[Encoding::PLAIN, Encoding::RLE],
- );
- check_encoding_write_support::<Int32Type>(
- WriterVersion::PARQUET_2_0,
- true,
- &[1, 2],
- Some(0),
- &[Encoding::PLAIN, Encoding::RLE_DICTIONARY, Encoding::RLE],
- );
- check_encoding_write_support::<Int32Type>(
- WriterVersion::PARQUET_2_0,
- false,
- &[1, 2],
- None,
- &[Encoding::DELTA_BINARY_PACKED, Encoding::RLE],
- );
- }
-
- #[test]
- fn test_column_writer_default_encoding_support_int64() {
- check_encoding_write_support::<Int64Type>(
- WriterVersion::PARQUET_1_0,
- true,
- &[1, 2],
- Some(0),
- &[Encoding::PLAIN, Encoding::RLE_DICTIONARY, Encoding::RLE],
- );
- check_encoding_write_support::<Int64Type>(
- WriterVersion::PARQUET_1_0,
- false,
- &[1, 2],
- None,
- &[Encoding::PLAIN, Encoding::RLE],
- );
- check_encoding_write_support::<Int64Type>(
- WriterVersion::PARQUET_2_0,
- true,
- &[1, 2],
- Some(0),
- &[Encoding::PLAIN, Encoding::RLE_DICTIONARY, Encoding::RLE],
- );
- check_encoding_write_support::<Int64Type>(
- WriterVersion::PARQUET_2_0,
- false,
- &[1, 2],
- None,
- &[Encoding::DELTA_BINARY_PACKED, Encoding::RLE],
- );
- }
-
- #[test]
- fn test_column_writer_default_encoding_support_int96() {
- check_encoding_write_support::<Int96Type>(
- WriterVersion::PARQUET_1_0,
- true,
- &[Int96::from(vec![1, 2, 3])],
- Some(0),
- &[Encoding::PLAIN, Encoding::RLE_DICTIONARY, Encoding::RLE],
- );
- check_encoding_write_support::<Int96Type>(
- WriterVersion::PARQUET_1_0,
- false,
- &[Int96::from(vec![1, 2, 3])],
- None,
- &[Encoding::PLAIN, Encoding::RLE],
- );
- check_encoding_write_support::<Int96Type>(
- WriterVersion::PARQUET_2_0,
- true,
- &[Int96::from(vec![1, 2, 3])],
- Some(0),
- &[Encoding::PLAIN, Encoding::RLE_DICTIONARY, Encoding::RLE],
- );
- check_encoding_write_support::<Int96Type>(
- WriterVersion::PARQUET_2_0,
- false,
- &[Int96::from(vec![1, 2, 3])],
- None,
- &[Encoding::PLAIN, Encoding::RLE],
- );
- }
-
- #[test]
- fn test_column_writer_default_encoding_support_float() {
- check_encoding_write_support::<FloatType>(
- WriterVersion::PARQUET_1_0,
- true,
- &[1.0, 2.0],
- Some(0),
- &[Encoding::PLAIN, Encoding::RLE_DICTIONARY, Encoding::RLE],
- );
- check_encoding_write_support::<FloatType>(
- WriterVersion::PARQUET_1_0,
- false,
- &[1.0, 2.0],
- None,
- &[Encoding::PLAIN, Encoding::RLE],
- );
- check_encoding_write_support::<FloatType>(
- WriterVersion::PARQUET_2_0,
- true,
- &[1.0, 2.0],
- Some(0),
- &[Encoding::PLAIN, Encoding::RLE_DICTIONARY, Encoding::RLE],
- );
- check_encoding_write_support::<FloatType>(
- WriterVersion::PARQUET_2_0,
- false,
- &[1.0, 2.0],
- None,
- &[Encoding::PLAIN, Encoding::RLE],
- );
- }
-
- #[test]
- fn test_column_writer_default_encoding_support_double() {
- check_encoding_write_support::<DoubleType>(
- WriterVersion::PARQUET_1_0,
- true,
- &[1.0, 2.0],
- Some(0),
- &[Encoding::PLAIN, Encoding::RLE_DICTIONARY, Encoding::RLE],
- );
- check_encoding_write_support::<DoubleType>(
- WriterVersion::PARQUET_1_0,
- false,
- &[1.0, 2.0],
- None,
- &[Encoding::PLAIN, Encoding::RLE],
- );
- check_encoding_write_support::<DoubleType>(
- WriterVersion::PARQUET_2_0,
- true,
- &[1.0, 2.0],
- Some(0),
- &[Encoding::PLAIN, Encoding::RLE_DICTIONARY, Encoding::RLE],
- );
- check_encoding_write_support::<DoubleType>(
- WriterVersion::PARQUET_2_0,
- false,
- &[1.0, 2.0],
- None,
- &[Encoding::PLAIN, Encoding::RLE],
- );
- }
-
- #[test]
- fn test_column_writer_default_encoding_support_byte_array() {
- check_encoding_write_support::<ByteArrayType>(
- WriterVersion::PARQUET_1_0,
- true,
- &[ByteArray::from(vec![1u8])],
- Some(0),
- &[Encoding::PLAIN, Encoding::RLE_DICTIONARY, Encoding::RLE],
- );
- check_encoding_write_support::<ByteArrayType>(
- WriterVersion::PARQUET_1_0,
- false,
- &[ByteArray::from(vec![1u8])],
- None,
- &[Encoding::PLAIN, Encoding::RLE],
- );
- check_encoding_write_support::<ByteArrayType>(
- WriterVersion::PARQUET_2_0,
- true,
- &[ByteArray::from(vec![1u8])],
- Some(0),
- &[Encoding::PLAIN, Encoding::RLE_DICTIONARY, Encoding::RLE],
- );
- check_encoding_write_support::<ByteArrayType>(
- WriterVersion::PARQUET_2_0,
- false,
- &[ByteArray::from(vec![1u8])],
- None,
- &[Encoding::DELTA_BYTE_ARRAY, Encoding::RLE],
- );
- }
-
- #[test]
- fn test_column_writer_default_encoding_support_fixed_len_byte_array() {
- check_encoding_write_support::<FixedLenByteArrayType>(
- WriterVersion::PARQUET_1_0,
- true,
- &[ByteArray::from(vec![1u8]).into()],
- None,
- &[Encoding::PLAIN, Encoding::RLE],
- );
- check_encoding_write_support::<FixedLenByteArrayType>(
- WriterVersion::PARQUET_1_0,
- false,
- &[ByteArray::from(vec![1u8]).into()],
- None,
- &[Encoding::PLAIN, Encoding::RLE],
- );
- check_encoding_write_support::<FixedLenByteArrayType>(
- WriterVersion::PARQUET_2_0,
- true,
- &[ByteArray::from(vec![1u8]).into()],
- Some(0),
- &[Encoding::PLAIN, Encoding::RLE_DICTIONARY, Encoding::RLE],
- );
- check_encoding_write_support::<FixedLenByteArrayType>(
- WriterVersion::PARQUET_2_0,
- false,
- &[ByteArray::from(vec![1u8]).into()],
- None,
- &[Encoding::DELTA_BYTE_ARRAY, Encoding::RLE],
- );
- }
-
- #[test]
- fn test_column_writer_check_metadata() {
- let page_writer = get_test_page_writer();
- let props = Arc::new(WriterProperties::builder().build());
- let mut writer = get_test_column_writer::<Int32Type>(page_writer, 0, 0, props);
- writer.write_batch(&[1, 2, 3, 4], None, None).unwrap();
-
- let (bytes_written, rows_written, metadata) = writer.close().unwrap();
- assert_eq!(bytes_written, 20);
- assert_eq!(rows_written, 4);
- assert_eq!(
- metadata.encodings(),
- &vec![Encoding::PLAIN, Encoding::RLE_DICTIONARY, Encoding::RLE]
- );
- assert_eq!(metadata.num_values(), 8); // dictionary + value indexes
- assert_eq!(metadata.compressed_size(), 20);
- assert_eq!(metadata.uncompressed_size(), 20);
- assert_eq!(metadata.data_page_offset(), 0);
- assert_eq!(metadata.dictionary_page_offset(), Some(0));
- if let Some(stats) = metadata.statistics() {
- assert!(stats.has_min_max_set());
- assert_eq!(stats.null_count(), 0);
- assert_eq!(stats.distinct_count(), None);
- if let Statistics::Int32(stats) = stats {
- assert_eq!(stats.min(), &1);
- assert_eq!(stats.max(), &4);
- } else {
- panic!("expecting Statistics::Int32");
- }
- } else {
- panic!("metadata missing statistics");
- }
- }
-
- #[test]
- fn test_column_writer_precalculated_statistics() {
- let page_writer = get_test_page_writer();
- let props = Arc::new(WriterProperties::builder().build());
- let mut writer = get_test_column_writer::<Int32Type>(page_writer, 0, 0, props);
- writer
- .write_batch_with_statistics(
- &[1, 2, 3, 4],
- None,
- None,
- &Some(-17),
- &Some(9000),
- Some(21),
- Some(55),
- )
- .unwrap();
-
- let (bytes_written, rows_written, metadata) = writer.close().unwrap();
- assert_eq!(bytes_written, 20);
- assert_eq!(rows_written, 4);
- assert_eq!(
- metadata.encodings(),
- &vec![Encoding::PLAIN, Encoding::RLE_DICTIONARY, Encoding::RLE]
- );
- assert_eq!(metadata.num_values(), 8); // dictionary + value indexes
- assert_eq!(metadata.compressed_size(), 20);
- assert_eq!(metadata.uncompressed_size(), 20);
- assert_eq!(metadata.data_page_offset(), 0);
- assert_eq!(metadata.dictionary_page_offset(), Some(0));
- if let Some(stats) = metadata.statistics() {
- assert!(stats.has_min_max_set());
- assert_eq!(stats.null_count(), 21);
- assert_eq!(stats.distinct_count().unwrap_or(0), 55);
- if let Statistics::Int32(stats) = stats {
- assert_eq!(stats.min(), &-17);
- assert_eq!(stats.max(), &9000);
- } else {
- panic!("expecting Statistics::Int32");
- }
- } else {
- panic!("metadata missing statistics");
- }
- }
-
- #[test]
- fn test_column_writer_empty_column_roundtrip() {
- let props = WriterProperties::builder().build();
- column_roundtrip::<Int32Type>("test_col_writer_rnd_1", props, &[], None, None);
- }
-
- #[test]
- fn test_column_writer_non_nullable_values_roundtrip() {
- let props = WriterProperties::builder().build();
- column_roundtrip_random::<Int32Type>(
- "test_col_writer_rnd_2",
- props,
- 1024,
- std::i32::MIN,
- std::i32::MAX,
- 0,
- 0,
- );
- }
-
- #[test]
- fn test_column_writer_nullable_non_repeated_values_roundtrip() {
- let props = WriterProperties::builder().build();
- column_roundtrip_random::<Int32Type>(
- "test_column_writer_nullable_non_repeated_values_roundtrip",
- props,
- 1024,
- std::i32::MIN,
- std::i32::MAX,
- 10,
- 0,
- );
- }
-
- #[test]
- fn test_column_writer_nullable_repeated_values_roundtrip() {
- let props = WriterProperties::builder().build();
- column_roundtrip_random::<Int32Type>(
- "test_col_writer_rnd_3",
- props,
- 1024,
- std::i32::MIN,
- std::i32::MAX,
- 10,
- 10,
- );
- }
-
- #[test]
- fn test_column_writer_dictionary_fallback_small_data_page() {
- let props = WriterProperties::builder()
- .set_dictionary_pagesize_limit(32)
- .set_data_pagesize_limit(32)
- .build();
- column_roundtrip_random::<Int32Type>(
- "test_col_writer_rnd_4",
- props,
- 1024,
- std::i32::MIN,
- std::i32::MAX,
- 10,
- 10,
- );
- }
-
- #[test]
- fn test_column_writer_small_write_batch_size() {
- for i in &[1usize, 2, 5, 10, 11, 1023] {
- let props = WriterProperties::builder().set_write_batch_size(*i).build();
-
- column_roundtrip_random::<Int32Type>(
- "test_col_writer_rnd_5",
- props,
- 1024,
- std::i32::MIN,
- std::i32::MAX,
- 10,
- 10,
- );
- }
- }
-
- #[test]
- fn test_column_writer_dictionary_disabled_v1() {
- let props = WriterProperties::builder()
- .set_writer_version(WriterVersion::PARQUET_1_0)
- .set_dictionary_enabled(false)
- .build();
- column_roundtrip_random::<Int32Type>(
- "test_col_writer_rnd_6",
- props,
- 1024,
- std::i32::MIN,
- std::i32::MAX,
- 10,
- 10,
- );
- }
-
- #[test]
- fn test_column_writer_dictionary_disabled_v2() {
- let props = WriterProperties::builder()
- .set_writer_version(WriterVersion::PARQUET_2_0)
- .set_dictionary_enabled(false)
- .build();
- column_roundtrip_random::<Int32Type>(
- "test_col_writer_rnd_7",
- props,
- 1024,
- std::i32::MIN,
- std::i32::MAX,
- 10,
- 10,
- );
- }
-
- #[test]
- fn test_column_writer_compression_v1() {
- let props = WriterProperties::builder()
- .set_writer_version(WriterVersion::PARQUET_1_0)
- .set_compression(Compression::SNAPPY)
- .build();
- column_roundtrip_random::<Int32Type>(
- "test_col_writer_rnd_8",
- props,
- 2048,
- std::i32::MIN,
- std::i32::MAX,
- 10,
- 10,
- );
- }
-
- #[test]
- fn test_column_writer_compression_v2() {
- let props = WriterProperties::builder()
- .set_writer_version(WriterVersion::PARQUET_2_0)
- .set_compression(Compression::SNAPPY)
- .build();
- column_roundtrip_random::<Int32Type>(
- "test_col_writer_rnd_9",
- props,
- 2048,
- std::i32::MIN,
- std::i32::MAX,
- 10,
- 10,
- );
- }
-
- #[test]
- fn test_column_writer_add_data_pages_with_dict() {
- // ARROW-5129: Test verifies that we add data page in case of dictionary encoding
- // and no fallback occurred so far.
- let file = get_temp_file("test_column_writer_add_data_pages_with_dict", &[]);
- let sink = FileSink::new(&file);
- let page_writer = Box::new(SerializedPageWriter::new(sink));
- let props = Arc::new(
- WriterProperties::builder()
- .set_data_pagesize_limit(15) // actually each page will have size 15-18 bytes
- .set_write_batch_size(3) // write 3 values at a time
- .build(),
- );
- let data = &[1, 2, 3, 4, 5, 6, 7, 8, 9, 10];
- let mut writer = get_test_column_writer::<Int32Type>(page_writer, 0, 0, props);
- writer.write_batch(data, None, None).unwrap();
- let (bytes_written, _, _) = writer.close().unwrap();
-
- // Read pages and check the sequence
- let source = FileSource::new(&file, 0, bytes_written as usize);
- let mut page_reader = Box::new(
- SerializedPageReader::new(
- source,
- data.len() as i64,
- Compression::UNCOMPRESSED,
- Int32Type::get_physical_type(),
- )
- .unwrap(),
- );
- let mut res = Vec::new();
- while let Some(page) = page_reader.get_next_page().unwrap() {
- res.push((page.page_type(), page.num_values()));
- }
- assert_eq!(
- res,
- vec![
- (PageType::DICTIONARY_PAGE, 10),
- (PageType::DATA_PAGE, 3),
- (PageType::DATA_PAGE, 3),
- (PageType::DATA_PAGE, 3),
- (PageType::DATA_PAGE, 1)
- ]
- );
- }
-
- #[test]
- fn test_float_statistics_nan_middle() {
- let stats = statistics_roundtrip::<FloatType>(&[1.0, f32::NAN, 2.0]);
- assert!(stats.has_min_max_set());
- if let Statistics::Float(stats) = stats {
- assert_eq!(stats.min(), &1.0);
- assert_eq!(stats.max(), &2.0);
- } else {
- panic!("expecting Statistics::Float");
- }
- }
-
- #[test]
- fn test_float_statistics_nan_start() {
- let stats = statistics_roundtrip::<FloatType>(&[f32::NAN, 1.0, 2.0]);
- assert!(stats.has_min_max_set());
- if let Statistics::Float(stats) = stats {
- assert_eq!(stats.min(), &1.0);
- assert_eq!(stats.max(), &2.0);
- } else {
- panic!("expecting Statistics::Float");
- }
- }
-
- #[test]
- fn test_float_statistics_nan_only() {
- let stats = statistics_roundtrip::<FloatType>(&[f32::NAN, f32::NAN]);
- assert!(!stats.has_min_max_set());
- assert!(matches!(stats, Statistics::Float(_)));
- }
-
- #[test]
- fn test_double_statistics_nan_middle() {
- let stats = statistics_roundtrip::<DoubleType>(&[1.0, f64::NAN, 2.0]);
- assert!(stats.has_min_max_set());
- if let Statistics::Double(stats) = stats {
- assert_eq!(stats.min(), &1.0);
- assert_eq!(stats.max(), &2.0);
- } else {
- panic!("expecting Statistics::Float");
- }
- }
-
- #[test]
- fn test_double_statistics_nan_start() {
- let stats = statistics_roundtrip::<DoubleType>(&[f64::NAN, 1.0, 2.0]);
- assert!(stats.has_min_max_set());
- if let Statistics::Double(stats) = stats {
- assert_eq!(stats.min(), &1.0);
- assert_eq!(stats.max(), &2.0);
- } else {
- panic!("expecting Statistics::Float");
- }
- }
-
- #[test]
- fn test_double_statistics_nan_only() {
- let stats = statistics_roundtrip::<DoubleType>(&[f64::NAN, f64::NAN]);
- assert!(!stats.has_min_max_set());
- assert!(matches!(stats, Statistics::Double(_)));
- }
-
- /// Performs write-read roundtrip with randomly generated values and levels.
- /// `max_size` is maximum number of values or levels (if `max_def_level` > 0) to write
- /// for a column.
- fn column_roundtrip_random<T: DataType>(
- file_name: &str,
- props: WriterProperties,
- max_size: usize,
- min_value: T::T,
- max_value: T::T,
- max_def_level: i16,
- max_rep_level: i16,
- ) where
- T::T: PartialOrd + SampleUniform + Copy,
- {
- let mut num_values: usize = 0;
-
- let mut buf: Vec<i16> = Vec::new();
- let def_levels = if max_def_level > 0 {
- random_numbers_range(max_size, 0, max_def_level + 1, &mut buf);
- for &dl in &buf[..] {
- if dl == max_def_level {
- num_values += 1;
- }
- }
- Some(&buf[..])
- } else {
- num_values = max_size;
- None
- };
-
- let mut buf: Vec<i16> = Vec::new();
- let rep_levels = if max_rep_level > 0 {
- random_numbers_range(max_size, 0, max_rep_level + 1, &mut buf);
- Some(&buf[..])
- } else {
- None
- };
-
- let mut values: Vec<T::T> = Vec::new();
- random_numbers_range(num_values, min_value, max_value, &mut values);
-
- column_roundtrip::<T>(file_name, props, &values[..], def_levels, rep_levels);
- }
-
- /// Performs write-read roundtrip and asserts written values and levels.
- fn column_roundtrip<'a, T: DataType>(
- file_name: &'a str,
- props: WriterProperties,
- values: &[T::T],
- def_levels: Option<&[i16]>,
- rep_levels: Option<&[i16]>,
- ) {
- let file = get_temp_file(file_name, &[]);
- let sink = FileSink::new(&file);
- let page_writer = Box::new(SerializedPageWriter::new(sink));
-
- let max_def_level = match def_levels {
- Some(buf) => *buf.iter().max().unwrap_or(&0i16),
- None => 0i16,
- };
-
- let max_rep_level = match rep_levels {
- Some(buf) => *buf.iter().max().unwrap_or(&0i16),
- None => 0i16,
- };
-
- let mut max_batch_size = values.len();
- if let Some(levels) = def_levels {
- max_batch_size = cmp::max(max_batch_size, levels.len());
- }
- if let Some(levels) = rep_levels {
- max_batch_size = cmp::max(max_batch_size, levels.len());
- }
-
- let mut writer = get_test_column_writer::<T>(
- page_writer,
- max_def_level,
- max_rep_level,
- Arc::new(props),
- );
-
- let values_written = writer.write_batch(values, def_levels, rep_levels).unwrap();
- assert_eq!(values_written, values.len());
- let (bytes_written, rows_written, column_metadata) = writer.close().unwrap();
-
- let source = FileSource::new(&file, 0, bytes_written as usize);
- let page_reader = Box::new(
- SerializedPageReader::new(
- source,
- column_metadata.num_values(),
- column_metadata.compression(),
- T::get_physical_type(),
- )
- .unwrap(),
- );
- let reader =
- get_test_column_reader::<T>(page_reader, max_def_level, max_rep_level);
-
- let mut actual_values = vec![T::T::default(); max_batch_size];
- let mut actual_def_levels = def_levels.map(|_| vec![0i16; max_batch_size]);
- let mut actual_rep_levels = rep_levels.map(|_| vec![0i16; max_batch_size]);
-
- let (values_read, levels_read) = read_fully(
- reader,
- max_batch_size,
- actual_def_levels.as_mut(),
- actual_rep_levels.as_mut(),
- actual_values.as_mut_slice(),
- );
-
- // Assert values, definition and repetition levels.
-
- assert_eq!(&actual_values[..values_read], values);
- match actual_def_levels {
- Some(ref vec) => assert_eq!(Some(&vec[..levels_read]), def_levels),
- None => assert_eq!(None, def_levels),
- }
- match actual_rep_levels {
- Some(ref vec) => assert_eq!(Some(&vec[..levels_read]), rep_levels),
- None => assert_eq!(None, rep_levels),
- }
-
- // Assert written rows.
-
- if let Some(levels) = actual_rep_levels {
- let mut actual_rows_written = 0;
- for l in levels {
- if l == 0 {
- actual_rows_written += 1;
- }
- }
- assert_eq!(actual_rows_written, rows_written);
- } else if actual_def_levels.is_some() {
- assert_eq!(levels_read as u64, rows_written);
- } else {
- assert_eq!(values_read as u64, rows_written);
- }
- }
-
- /// Performs write of provided values and returns column metadata of those values.
- /// Used to test encoding support for column writer.
- fn column_write_and_get_metadata<T: DataType>(
- props: WriterProperties,
- values: &[T::T],
- ) -> ColumnChunkMetaData {
- let page_writer = get_test_page_writer();
- let props = Arc::new(props);
- let mut writer = get_test_column_writer::<T>(page_writer, 0, 0, props);
- writer.write_batch(values, None, None).unwrap();
- let (_, _, metadata) = writer.close().unwrap();
- metadata
- }
-
- // Function to use in tests for EncodingWriteSupport. This checks that dictionary
- // offset and encodings to make sure that column writer uses provided by trait
- // encodings.
- fn check_encoding_write_support<T: DataType>(
- version: WriterVersion,
- dict_enabled: bool,
- data: &[T::T],
- dictionary_page_offset: Option<i64>,
- encodings: &[Encoding],
- ) {
- let props = WriterProperties::builder()
- .set_writer_version(version)
- .set_dictionary_enabled(dict_enabled)
- .build();
- let meta = column_write_and_get_metadata::<T>(props, data);
- assert_eq!(meta.dictionary_page_offset(), dictionary_page_offset);
- assert_eq!(meta.encodings(), &encodings);
- }
-
- /// Reads one batch of data, considering that batch is large enough to capture all of
- /// the values and levels.
- fn read_fully<T: DataType>(
- mut reader: ColumnReaderImpl<T>,
- batch_size: usize,
- mut def_levels: Option<&mut Vec<i16>>,
- mut rep_levels: Option<&mut Vec<i16>>,
- values: &mut [T::T],
- ) -> (usize, usize) {
- let actual_def_levels = def_levels.as_mut().map(|vec| &mut vec[..]);
- let actual_rep_levels = rep_levels.as_mut().map(|vec| &mut vec[..]);
- reader
- .read_batch(batch_size, actual_def_levels, actual_rep_levels, values)
- .unwrap()
- }
-
- /// Returns column writer.
- fn get_test_column_writer<T: DataType>(
- page_writer: Box<dyn PageWriter>,
- max_def_level: i16,
- max_rep_level: i16,
- props: WriterPropertiesPtr,
- ) -> ColumnWriterImpl<T> {
- let descr = Arc::new(get_test_column_descr::<T>(max_def_level, max_rep_level));
- let column_writer = get_column_writer(descr, props, page_writer);
- get_typed_column_writer::<T>(column_writer)
- }
-
- /// Returns column reader.
- fn get_test_column_reader<T: DataType>(
- page_reader: Box<dyn PageReader>,
- max_def_level: i16,
- max_rep_level: i16,
- ) -> ColumnReaderImpl<T> {
- let descr = Arc::new(get_test_column_descr::<T>(max_def_level, max_rep_level));
- let column_reader = get_column_reader(descr, page_reader);
- get_typed_column_reader::<T>(column_reader)
- }
-
- /// Returns descriptor for primitive column.
- fn get_test_column_descr<T: DataType>(
- max_def_level: i16,
- max_rep_level: i16,
- ) -> ColumnDescriptor {
- let path = ColumnPath::from("col");
- let tpe = SchemaType::primitive_type_builder("col", T::get_physical_type())
- // length is set for "encoding support" tests for FIXED_LEN_BYTE_ARRAY type,
- // it should be no-op for other types
- .with_length(1)
- .build()
- .unwrap();
- ColumnDescriptor::new(Arc::new(tpe), max_def_level, max_rep_level, path)
- }
-
- /// Returns page writer that collects pages without serializing them.
- fn get_test_page_writer() -> Box<dyn PageWriter> {
- Box::new(TestPageWriter {})
- }
-
- struct TestPageWriter {}
-
- impl PageWriter for TestPageWriter {
- fn write_page(&mut self, page: CompressedPage) -> Result<PageWriteSpec> {
- let mut res = PageWriteSpec::new();
- res.page_type = page.page_type();
- res.uncompressed_size = page.uncompressed_size();
- res.compressed_size = page.compressed_size();
- res.num_values = page.num_values();
- res.offset = 0;
- res.bytes_written = page.data().len() as u64;
- Ok(res)
- }
-
- fn write_metadata(&mut self, _metadata: &ColumnChunkMetaData) -> Result<()> {
- Ok(())
- }
-
- fn close(&mut self) -> Result<()> {
- Ok(())
- }
- }
-
- /// Write data into parquet using [`get_test_page_writer`] and [`get_test_column_writer`] and returns generated statistics.
- fn statistics_roundtrip<T: DataType>(values: &[<T as DataType>::T]) -> Statistics {
- let page_writer = get_test_page_writer();
- let props = Arc::new(WriterProperties::builder().build());
- let mut writer = get_test_column_writer::<T>(page_writer, 0, 0, props);
- writer.write_batch(values, None, None).unwrap();
-
- let (_bytes_written, _rows_written, metadata) = writer.close().unwrap();
- if let Some(stats) = metadata.statistics() {
- stats.clone()
- } else {
- panic!("metadata missing statistics");
- }
- }
-}
diff --git a/parquet/src/compression.rs b/parquet/src/compression.rs
deleted file mode 100644
index a115597..0000000
--- a/parquet/src/compression.rs
+++ /dev/null
@@ -1,393 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-//! Contains codec interface and supported codec implementations.
-//!
-//! See [`Compression`](crate::basic::Compression) enum for all available compression
-//! algorithms.
-//!
-//! # Example
-//!
-//! ```no_run
-//! use parquet::{basic::Compression, compression::create_codec};
-//!
-//! let mut codec = match create_codec(Compression::SNAPPY) {
-//! Ok(Some(codec)) => codec,
-//! _ => panic!(),
-//! };
-//!
-//! let data = vec![b'p', b'a', b'r', b'q', b'u', b'e', b't'];
-//! let mut compressed = vec![];
-//! codec.compress(&data[..], &mut compressed).unwrap();
-//!
-//! let mut output = vec![];
-//! codec.decompress(&compressed[..], &mut output).unwrap();
-//!
-//! assert_eq!(output, data);
-//! ```
-
-use crate::basic::Compression as CodecType;
-use crate::errors::{ParquetError, Result};
-
-/// Parquet compression codec interface.
-pub trait Codec {
- /// Compresses data stored in slice `input_buf` and writes the compressed result
- /// to `output_buf`.
- /// Note that you'll need to call `clear()` before reusing the same `output_buf`
- /// across different `compress` calls.
- fn compress(&mut self, input_buf: &[u8], output_buf: &mut Vec<u8>) -> Result<()>;
-
- /// Decompresses data stored in slice `input_buf` and writes output to `output_buf`.
- /// Returns the total number of bytes written.
- fn decompress(&mut self, input_buf: &[u8], output_buf: &mut Vec<u8>)
- -> Result<usize>;
-}
-
-/// Given the compression type `codec`, returns a codec used to compress and decompress
-/// bytes for the compression type.
-/// This returns `None` if the codec type is `UNCOMPRESSED`.
-pub fn create_codec(codec: CodecType) -> Result<Option<Box<dyn Codec>>> {
- match codec {
- #[cfg(any(feature = "brotli", test))]
- CodecType::BROTLI => Ok(Some(Box::new(BrotliCodec::new()))),
- #[cfg(any(feature = "flate2", test))]
- CodecType::GZIP => Ok(Some(Box::new(GZipCodec::new()))),
- #[cfg(any(feature = "snap", test))]
- CodecType::SNAPPY => Ok(Some(Box::new(SnappyCodec::new()))),
- #[cfg(any(feature = "lz4", test))]
- CodecType::LZ4 => Ok(Some(Box::new(LZ4Codec::new()))),
- #[cfg(any(feature = "zstd", test))]
- CodecType::ZSTD => Ok(Some(Box::new(ZSTDCodec::new()))),
- CodecType::UNCOMPRESSED => Ok(None),
- _ => Err(nyi_err!("The codec type {} is not supported yet", codec)),
- }
-}
-
-#[cfg(any(feature = "snap", test))]
-mod snappy_codec {
- use snap::raw::{decompress_len, max_compress_len, Decoder, Encoder};
-
- use crate::compression::Codec;
- use crate::errors::Result;
-
- /// Codec for Snappy compression format.
- pub struct SnappyCodec {
- decoder: Decoder,
- encoder: Encoder,
- }
-
- impl SnappyCodec {
- /// Creates new Snappy compression codec.
- pub(crate) fn new() -> Self {
- Self {
- decoder: Decoder::new(),
- encoder: Encoder::new(),
- }
- }
- }
-
- impl Codec for SnappyCodec {
- fn decompress(
- &mut self,
- input_buf: &[u8],
- output_buf: &mut Vec<u8>,
- ) -> Result<usize> {
- let len = decompress_len(input_buf)?;
- output_buf.resize(len, 0);
- self.decoder
- .decompress(input_buf, output_buf)
- .map_err(|e| e.into())
- }
-
- fn compress(&mut self, input_buf: &[u8], output_buf: &mut Vec<u8>) -> Result<()> {
- let output_buf_len = output_buf.len();
- let required_len = max_compress_len(input_buf.len());
- output_buf.resize(output_buf_len + required_len, 0);
- let n = self
- .encoder
- .compress(input_buf, &mut output_buf[output_buf_len..])?;
- output_buf.truncate(output_buf_len + n);
- Ok(())
- }
- }
-}
-#[cfg(any(feature = "snap", test))]
-pub use snappy_codec::*;
-
-#[cfg(any(feature = "flate2", test))]
-mod gzip_codec {
-
- use std::io::{Read, Write};
-
- use flate2::{read, write, Compression};
-
- use crate::compression::Codec;
- use crate::errors::Result;
-
- /// Codec for GZIP compression algorithm.
- pub struct GZipCodec {}
-
- impl GZipCodec {
- /// Creates new GZIP compression codec.
- pub(crate) fn new() -> Self {
- Self {}
- }
- }
-
- impl Codec for GZipCodec {
- fn decompress(
- &mut self,
- input_buf: &[u8],
- output_buf: &mut Vec<u8>,
- ) -> Result<usize> {
- let mut decoder = read::GzDecoder::new(input_buf);
- decoder.read_to_end(output_buf).map_err(|e| e.into())
- }
-
- fn compress(&mut self, input_buf: &[u8], output_buf: &mut Vec<u8>) -> Result<()> {
- let mut encoder = write::GzEncoder::new(output_buf, Compression::default());
- encoder.write_all(input_buf)?;
- encoder.try_finish().map_err(|e| e.into())
- }
- }
-}
-#[cfg(any(feature = "flate2", test))]
-pub use gzip_codec::*;
-
-#[cfg(any(feature = "brotli", test))]
-mod brotli_codec {
-
- use std::io::{Read, Write};
-
- use crate::compression::Codec;
- use crate::errors::Result;
-
- const BROTLI_DEFAULT_BUFFER_SIZE: usize = 4096;
- const BROTLI_DEFAULT_COMPRESSION_QUALITY: u32 = 1; // supported levels 0-9
- const BROTLI_DEFAULT_LG_WINDOW_SIZE: u32 = 22; // recommended between 20-22
-
- /// Codec for Brotli compression algorithm.
- pub struct BrotliCodec {}
-
- impl BrotliCodec {
- /// Creates new Brotli compression codec.
- pub(crate) fn new() -> Self {
- Self {}
- }
- }
-
- impl Codec for BrotliCodec {
- fn decompress(
- &mut self,
- input_buf: &[u8],
- output_buf: &mut Vec<u8>,
- ) -> Result<usize> {
- brotli::Decompressor::new(input_buf, BROTLI_DEFAULT_BUFFER_SIZE)
- .read_to_end(output_buf)
- .map_err(|e| e.into())
- }
-
- fn compress(&mut self, input_buf: &[u8], output_buf: &mut Vec<u8>) -> Result<()> {
- let mut encoder = brotli::CompressorWriter::new(
- output_buf,
- BROTLI_DEFAULT_BUFFER_SIZE,
- BROTLI_DEFAULT_COMPRESSION_QUALITY,
- BROTLI_DEFAULT_LG_WINDOW_SIZE,
- );
- encoder.write_all(input_buf)?;
- encoder.flush().map_err(|e| e.into())
- }
- }
-}
-#[cfg(any(feature = "brotli", test))]
-pub use brotli_codec::*;
-
-#[cfg(any(feature = "lz4", test))]
-mod lz4_codec {
- use std::io::{Read, Write};
-
- use crate::compression::Codec;
- use crate::errors::Result;
-
- const LZ4_BUFFER_SIZE: usize = 4096;
-
- /// Codec for LZ4 compression algorithm.
- pub struct LZ4Codec {}
-
- impl LZ4Codec {
- /// Creates new LZ4 compression codec.
- pub(crate) fn new() -> Self {
- Self {}
- }
- }
-
- impl Codec for LZ4Codec {
- fn decompress(
- &mut self,
- input_buf: &[u8],
- output_buf: &mut Vec<u8>,
- ) -> Result<usize> {
- let mut decoder = lz4::Decoder::new(input_buf)?;
- let mut buffer: [u8; LZ4_BUFFER_SIZE] = [0; LZ4_BUFFER_SIZE];
- let mut total_len = 0;
- loop {
- let len = decoder.read(&mut buffer)?;
- if len == 0 {
- break;
- }
- total_len += len;
- output_buf.write_all(&buffer[0..len])?;
- }
- Ok(total_len)
- }
-
- fn compress(&mut self, input_buf: &[u8], output_buf: &mut Vec<u8>) -> Result<()> {
- let mut encoder = lz4::EncoderBuilder::new().build(output_buf)?;
- let mut from = 0;
- loop {
- let to = std::cmp::min(from + LZ4_BUFFER_SIZE, input_buf.len());
- encoder.write_all(&input_buf[from..to])?;
- from += LZ4_BUFFER_SIZE;
- if from >= input_buf.len() {
- break;
- }
- }
- encoder.finish().1.map_err(|e| e.into())
- }
- }
-}
-#[cfg(any(feature = "lz4", test))]
-pub use lz4_codec::*;
-
-#[cfg(any(feature = "zstd", test))]
-mod zstd_codec {
- use std::io::{self, Write};
-
- use crate::compression::Codec;
- use crate::errors::Result;
-
- /// Codec for Zstandard compression algorithm.
- pub struct ZSTDCodec {}
-
- impl ZSTDCodec {
- /// Creates new Zstandard compression codec.
- pub(crate) fn new() -> Self {
- Self {}
- }
- }
-
- /// Compression level (1-21) for ZSTD. Choose 1 here for better compression speed.
- const ZSTD_COMPRESSION_LEVEL: i32 = 1;
-
- impl Codec for ZSTDCodec {
- fn decompress(
- &mut self,
- input_buf: &[u8],
- output_buf: &mut Vec<u8>,
- ) -> Result<usize> {
- let mut decoder = zstd::Decoder::new(input_buf)?;
- match io::copy(&mut decoder, output_buf) {
- Ok(n) => Ok(n as usize),
- Err(e) => Err(e.into()),
- }
- }
-
- fn compress(&mut self, input_buf: &[u8], output_buf: &mut Vec<u8>) -> Result<()> {
- let mut encoder = zstd::Encoder::new(output_buf, ZSTD_COMPRESSION_LEVEL)?;
- encoder.write_all(input_buf)?;
- match encoder.finish() {
- Ok(_) => Ok(()),
- Err(e) => Err(e.into()),
- }
- }
- }
-}
-#[cfg(any(feature = "zstd", test))]
-pub use zstd_codec::*;
-
-#[cfg(test)]
-mod tests {
- use super::*;
-
- use crate::util::test_common::*;
-
- fn test_roundtrip(c: CodecType, data: &[u8]) {
- let mut c1 = create_codec(c).unwrap().unwrap();
- let mut c2 = create_codec(c).unwrap().unwrap();
-
- // Compress with c1
- let mut compressed = Vec::new();
- let mut decompressed = Vec::new();
- c1.compress(data, &mut compressed)
- .expect("Error when compressing");
-
- // Decompress with c2
- let mut decompressed_size = c2
- .decompress(compressed.as_slice(), &mut decompressed)
- .expect("Error when decompressing");
- assert_eq!(data.len(), decompressed_size);
- decompressed.truncate(decompressed_size);
- assert_eq!(data, decompressed.as_slice());
-
- compressed.clear();
-
- // Compress with c2
- c2.compress(data, &mut compressed)
- .expect("Error when compressing");
-
- // Decompress with c1
- decompressed_size = c1
- .decompress(compressed.as_slice(), &mut decompressed)
- .expect("Error when decompressing");
- assert_eq!(data.len(), decompressed_size);
- decompressed.truncate(decompressed_size);
- assert_eq!(data, decompressed.as_slice());
- }
-
- fn test_codec(c: CodecType) {
- let sizes = vec![100, 10000, 100000];
- for size in sizes {
- let data = random_bytes(size);
- test_roundtrip(c, &data);
- }
- }
-
- #[test]
- fn test_codec_snappy() {
- test_codec(CodecType::SNAPPY);
- }
-
- #[test]
- fn test_codec_gzip() {
- test_codec(CodecType::GZIP);
- }
-
- #[test]
- fn test_codec_brotli() {
- test_codec(CodecType::BROTLI);
- }
-
- #[test]
- fn test_codec_lz4() {
- test_codec(CodecType::LZ4);
- }
-
- #[test]
- fn test_codec_zstd() {
- test_codec(CodecType::ZSTD);
- }
-}
diff --git a/parquet/src/data_type.rs b/parquet/src/data_type.rs
deleted file mode 100644
index f97df3c..0000000
--- a/parquet/src/data_type.rs
+++ /dev/null
@@ -1,1365 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-//! Data types that connect Parquet physical types with their Rust-specific
-//! representations.
-use std::cmp::Ordering;
-use std::fmt;
-use std::mem;
-use std::ops::{Deref, DerefMut};
-use std::str::from_utf8;
-
-use byteorder::{BigEndian, ByteOrder};
-
-use crate::basic::Type;
-use crate::column::reader::{ColumnReader, ColumnReaderImpl};
-use crate::column::writer::{ColumnWriter, ColumnWriterImpl};
-use crate::errors::{ParquetError, Result};
-use crate::util::{
- bit_util::{from_ne_slice, FromBytes},
- memory::{ByteBuffer, ByteBufferPtr},
-};
-
-/// Rust representation for logical type INT96, value is backed by an array of `u32`.
-/// The type only takes 12 bytes, without extra padding.
-#[derive(Clone, Debug, PartialOrd)]
-pub struct Int96 {
- value: Option<[u32; 3]>,
-}
-
-impl Int96 {
- /// Creates new INT96 type struct with no data set.
- pub fn new() -> Self {
- Self { value: None }
- }
-
- /// Returns underlying data as slice of [`u32`].
- #[inline]
- pub fn data(&self) -> &[u32] {
- self.value
- .as_ref()
- .expect("set_data should have been called")
- }
-
- /// Sets data for this INT96 type.
- #[inline]
- pub fn set_data(&mut self, elem0: u32, elem1: u32, elem2: u32) {
- self.value = Some([elem0, elem1, elem2]);
- }
-
- /// Converts this INT96 into an i64 representing the number of MILLISECONDS since Epoch
- pub fn to_i64(&self) -> i64 {
- const JULIAN_DAY_OF_EPOCH: i64 = 2_440_588;
- const SECONDS_PER_DAY: i64 = 86_400;
- const MILLIS_PER_SECOND: i64 = 1_000;
-
- let day = self.data()[2] as i64;
- let nanoseconds = ((self.data()[1] as i64) << 32) + self.data()[0] as i64;
- let seconds = (day - JULIAN_DAY_OF_EPOCH) * SECONDS_PER_DAY;
-
- seconds * MILLIS_PER_SECOND + nanoseconds / 1_000_000
- }
-}
-
-impl Default for Int96 {
- fn default() -> Self {
- Self { value: None }
- }
-}
-
-impl PartialEq for Int96 {
- fn eq(&self, other: &Int96) -> bool {
- match (&self.value, &other.value) {
- (Some(v1), Some(v2)) => v1 == v2,
- (None, None) => true,
- _ => false,
- }
- }
-}
-
-impl From<Vec<u32>> for Int96 {
- fn from(buf: Vec<u32>) -> Self {
- assert_eq!(buf.len(), 3);
- let mut result = Self::new();
- result.set_data(buf[0], buf[1], buf[2]);
- result
- }
-}
-
-impl fmt::Display for Int96 {
- #[cold]
- fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
- write!(f, "{:?}", self.data())
- }
-}
-
-/// Rust representation for BYTE_ARRAY and FIXED_LEN_BYTE_ARRAY Parquet physical types.
-/// Value is backed by a byte buffer.
-#[derive(Clone, Debug)]
-pub struct ByteArray {
- data: Option<ByteBufferPtr>,
-}
-
-impl PartialOrd for ByteArray {
- fn partial_cmp(&self, other: &ByteArray) -> Option<Ordering> {
- if self.data.is_some() && other.data.is_some() {
- match self.len().cmp(&other.len()) {
- Ordering::Greater => Some(Ordering::Greater),
- Ordering::Less => Some(Ordering::Less),
- Ordering::Equal => {
- for (v1, v2) in self.data().iter().zip(other.data().iter()) {
- match v1.cmp(v2) {
- Ordering::Greater => return Some(Ordering::Greater),
- Ordering::Less => return Some(Ordering::Less),
- _ => {}
- }
- }
- Some(Ordering::Equal)
- }
- }
- } else {
- None
- }
- }
-}
-
-impl ByteArray {
- /// Creates new byte array with no data set.
- #[inline]
- pub fn new() -> Self {
- ByteArray { data: None }
- }
-
- /// Gets length of the underlying byte buffer.
- #[inline]
- pub fn len(&self) -> usize {
- assert!(self.data.is_some());
- self.data.as_ref().unwrap().len()
- }
-
- /// Checks if the underlying buffer is empty.
- #[inline]
- pub fn is_empty(&self) -> bool {
- self.len() == 0
- }
-
- /// Returns slice of data.
- #[inline]
- pub fn data(&self) -> &[u8] {
- self.data
- .as_ref()
- .expect("set_data should have been called")
- .as_ref()
- }
-
- /// Set data from another byte buffer.
- #[inline]
- pub fn set_data(&mut self, data: ByteBufferPtr) {
- self.data = Some(data);
- }
-
- /// Returns `ByteArray` instance with slice of values for a data.
- #[inline]
- pub fn slice(&self, start: usize, len: usize) -> Self {
- Self::from(
- self.data
- .as_ref()
- .expect("set_data should have been called")
- .range(start, len),
- )
- }
-
- pub fn as_utf8(&self) -> Result<&str> {
- self.data
- .as_ref()
- .map(|ptr| ptr.as_ref())
- .ok_or_else(|| general_err!("Can't convert empty byte array to utf8"))
- .and_then(|bytes| from_utf8(bytes).map_err(|e| e.into()))
- }
-}
-
-impl From<Vec<u8>> for ByteArray {
- fn from(buf: Vec<u8>) -> ByteArray {
- Self {
- data: Some(ByteBufferPtr::new(buf)),
- }
- }
-}
-
-impl<'a> From<&'a str> for ByteArray {
- fn from(s: &'a str) -> ByteArray {
- let mut v = Vec::new();
- v.extend_from_slice(s.as_bytes());
- Self {
- data: Some(ByteBufferPtr::new(v)),
- }
- }
-}
-
-impl From<ByteBufferPtr> for ByteArray {
- fn from(ptr: ByteBufferPtr) -> ByteArray {
- Self { data: Some(ptr) }
- }
-}
-
-impl From<ByteBuffer> for ByteArray {
- fn from(mut buf: ByteBuffer) -> ByteArray {
- Self {
- data: Some(buf.consume()),
- }
- }
-}
-
-impl Default for ByteArray {
- fn default() -> Self {
- ByteArray { data: None }
- }
-}
-
-impl PartialEq for ByteArray {
- fn eq(&self, other: &ByteArray) -> bool {
- match (&self.data, &other.data) {
- (Some(d1), Some(d2)) => d1.as_ref() == d2.as_ref(),
- (None, None) => true,
- _ => false,
- }
- }
-}
-
-impl fmt::Display for ByteArray {
- fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
- write!(f, "{:?}", self.data())
- }
-}
-
-/// Wrapper type for performance reasons, this represents `FIXED_LEN_BYTE_ARRAY` but in all other
-/// considerations behaves the same as `ByteArray`
-///
-/// # Performance notes:
-/// This type is a little unfortunate, without it the compiler generates code that takes quite a
-/// big hit on the CPU pipeline. Essentially the previous version stalls awaiting the result of
-/// `T::get_physical_type() == Type::FIXED_LEN_BYTE_ARRAY`.
-///
-/// Its debatable if this is wanted, it is out of spec for what parquet documents as its base
-/// types, although there are code paths in the Rust (and potentially the C++) versions that
-/// warrant this.
-///
-/// With this wrapper type the compiler generates more targetted code paths matching the higher
-/// level logical types, removing the data-hazard from all decoding and encoding paths.
-#[repr(transparent)]
-#[derive(Clone, Debug, Default)]
-pub struct FixedLenByteArray(ByteArray);
-
-impl PartialEq for FixedLenByteArray {
- fn eq(&self, other: &FixedLenByteArray) -> bool {
- self.0.eq(&other.0)
- }
-}
-
-impl PartialEq<ByteArray> for FixedLenByteArray {
- fn eq(&self, other: &ByteArray) -> bool {
- self.0.eq(other)
- }
-}
-
-impl PartialEq<FixedLenByteArray> for ByteArray {
- fn eq(&self, other: &FixedLenByteArray) -> bool {
- self.eq(&other.0)
- }
-}
-
-impl fmt::Display for FixedLenByteArray {
- fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
- self.0.fmt(f)
- }
-}
-
-impl PartialOrd for FixedLenByteArray {
- fn partial_cmp(&self, other: &FixedLenByteArray) -> Option<Ordering> {
- self.0.partial_cmp(&other.0)
- }
-}
-
-impl PartialOrd<FixedLenByteArray> for ByteArray {
- fn partial_cmp(&self, other: &FixedLenByteArray) -> Option<Ordering> {
- self.partial_cmp(&other.0)
- }
-}
-
-impl PartialOrd<ByteArray> for FixedLenByteArray {
- fn partial_cmp(&self, other: &ByteArray) -> Option<Ordering> {
- self.0.partial_cmp(other)
- }
-}
-
-impl Deref for FixedLenByteArray {
- type Target = ByteArray;
-
- fn deref(&self) -> &Self::Target {
- &self.0
- }
-}
-
-impl DerefMut for FixedLenByteArray {
- fn deref_mut(&mut self) -> &mut Self::Target {
- &mut self.0
- }
-}
-
-impl From<ByteArray> for FixedLenByteArray {
- fn from(other: ByteArray) -> Self {
- Self(other)
- }
-}
-
-impl From<FixedLenByteArray> for ByteArray {
- fn from(other: FixedLenByteArray) -> Self {
- other.0
- }
-}
-
-/// Rust representation for Decimal values.
-///
-/// This is not a representation of Parquet physical type, but rather a wrapper for
-/// DECIMAL logical type, and serves as container for raw parts of decimal values:
-/// unscaled value in bytes, precision and scale.
-#[derive(Clone, Debug)]
-pub enum Decimal {
- /// Decimal backed by `i32`.
- Int32 {
- value: [u8; 4],
- precision: i32,
- scale: i32,
- },
- /// Decimal backed by `i64`.
- Int64 {
- value: [u8; 8],
- precision: i32,
- scale: i32,
- },
- /// Decimal backed by byte array.
- Bytes {
- value: ByteArray,
- precision: i32,
- scale: i32,
- },
-}
-
-impl Decimal {
- /// Creates new decimal value from `i32`.
- pub fn from_i32(value: i32, precision: i32, scale: i32) -> Self {
- let mut bytes = [0; 4];
- BigEndian::write_i32(&mut bytes, value);
- Decimal::Int32 {
- value: bytes,
- precision,
- scale,
- }
- }
-
- /// Creates new decimal value from `i64`.
- pub fn from_i64(value: i64, precision: i32, scale: i32) -> Self {
- let mut bytes = [0; 8];
- BigEndian::write_i64(&mut bytes, value);
- Decimal::Int64 {
- value: bytes,
- precision,
- scale,
- }
- }
-
- /// Creates new decimal value from `ByteArray`.
- pub fn from_bytes(value: ByteArray, precision: i32, scale: i32) -> Self {
- Decimal::Bytes {
- value,
- precision,
- scale,
- }
- }
-
- /// Returns bytes of unscaled value.
- pub fn data(&self) -> &[u8] {
- match *self {
- Decimal::Int32 { ref value, .. } => value,
- Decimal::Int64 { ref value, .. } => value,
- Decimal::Bytes { ref value, .. } => value.data(),
- }
- }
-
- /// Returns decimal precision.
- pub fn precision(&self) -> i32 {
- match *self {
- Decimal::Int32 { precision, .. } => precision,
- Decimal::Int64 { precision, .. } => precision,
- Decimal::Bytes { precision, .. } => precision,
- }
- }
-
- /// Returns decimal scale.
- pub fn scale(&self) -> i32 {
- match *self {
- Decimal::Int32 { scale, .. } => scale,
- Decimal::Int64 { scale, .. } => scale,
- Decimal::Bytes { scale, .. } => scale,
- }
- }
-}
-
-impl Default for Decimal {
- fn default() -> Self {
- Self::from_i32(0, 0, 0)
- }
-}
-
-impl PartialEq for Decimal {
- fn eq(&self, other: &Decimal) -> bool {
- self.precision() == other.precision()
- && self.scale() == other.scale()
- && self.data() == other.data()
- }
-}
-
-/// Converts an instance of data type to a slice of bytes as `u8`.
-pub trait AsBytes {
- /// Returns slice of bytes for this data type.
- fn as_bytes(&self) -> &[u8];
-}
-
-/// Converts an slice of a data type to a slice of bytes.
-pub trait SliceAsBytes: Sized {
- /// Returns slice of bytes for a slice of this data type.
- fn slice_as_bytes(self_: &[Self]) -> &[u8];
- /// Return the internal representation as a mutable slice
- ///
- /// # Safety
- /// If modified you are _required_ to ensure the internal representation
- /// is valid and correct for the actual raw data
- unsafe fn slice_as_bytes_mut(self_: &mut [Self]) -> &mut [u8];
-}
-
-impl AsBytes for [u8] {
- fn as_bytes(&self) -> &[u8] {
- self
- }
-}
-
-macro_rules! gen_as_bytes {
- ($source_ty:ident) => {
- impl AsBytes for $source_ty {
- #[allow(clippy::size_of_in_element_count)]
- fn as_bytes(&self) -> &[u8] {
- unsafe {
- std::slice::from_raw_parts(
- self as *const $source_ty as *const u8,
- std::mem::size_of::<$source_ty>(),
- )
- }
- }
- }
-
- impl SliceAsBytes for $source_ty {
- #[inline]
- #[allow(clippy::size_of_in_element_count)]
- fn slice_as_bytes(self_: &[Self]) -> &[u8] {
- unsafe {
- std::slice::from_raw_parts(
- self_.as_ptr() as *const u8,
- std::mem::size_of::<$source_ty>() * self_.len(),
- )
- }
- }
-
- #[inline]
- #[allow(clippy::size_of_in_element_count)]
- unsafe fn slice_as_bytes_mut(self_: &mut [Self]) -> &mut [u8] {
- std::slice::from_raw_parts_mut(
- self_.as_mut_ptr() as *mut u8,
- std::mem::size_of::<$source_ty>() * self_.len(),
- )
- }
- }
- };
-}
-
-gen_as_bytes!(i8);
-gen_as_bytes!(i16);
-gen_as_bytes!(i32);
-gen_as_bytes!(i64);
-gen_as_bytes!(u8);
-gen_as_bytes!(u16);
-gen_as_bytes!(u32);
-gen_as_bytes!(u64);
-gen_as_bytes!(f32);
-gen_as_bytes!(f64);
-
-macro_rules! unimplemented_slice_as_bytes {
- ($ty: ty) => {
- impl SliceAsBytes for $ty {
- fn slice_as_bytes(_self: &[Self]) -> &[u8] {
- unimplemented!()
- }
-
- unsafe fn slice_as_bytes_mut(_self: &mut [Self]) -> &mut [u8] {
- unimplemented!()
- }
- }
- };
-}
-
-// TODO - Can Int96 and bool be implemented in these terms?
-unimplemented_slice_as_bytes!(Int96);
-unimplemented_slice_as_bytes!(bool);
-unimplemented_slice_as_bytes!(ByteArray);
-unimplemented_slice_as_bytes!(FixedLenByteArray);
-
-impl AsBytes for bool {
- fn as_bytes(&self) -> &[u8] {
- unsafe { std::slice::from_raw_parts(self as *const bool as *const u8, 1) }
- }
-}
-
-impl AsBytes for Int96 {
- fn as_bytes(&self) -> &[u8] {
- unsafe {
- std::slice::from_raw_parts(self.data() as *const [u32] as *const u8, 12)
- }
- }
-}
-
-impl AsBytes for ByteArray {
- fn as_bytes(&self) -> &[u8] {
- self.data()
- }
-}
-
-impl AsBytes for FixedLenByteArray {
- fn as_bytes(&self) -> &[u8] {
- self.data()
- }
-}
-
-impl AsBytes for Decimal {
- fn as_bytes(&self) -> &[u8] {
- self.data()
- }
-}
-
-impl AsBytes for Vec<u8> {
- fn as_bytes(&self) -> &[u8] {
- self.as_slice()
- }
-}
-
-impl<'a> AsBytes for &'a str {
- fn as_bytes(&self) -> &[u8] {
- (self as &str).as_bytes()
- }
-}
-
-impl AsBytes for str {
- fn as_bytes(&self) -> &[u8] {
- (self as &str).as_bytes()
- }
-}
-
-pub(crate) mod private {
- use crate::encodings::decoding::PlainDecoderDetails;
- use crate::util::bit_util::{BitReader, BitWriter};
- use crate::util::memory::ByteBufferPtr;
-
- use byteorder::ByteOrder;
- use std::convert::TryInto;
-
- use super::{ParquetError, Result, SliceAsBytes};
-
- pub type BitIndex = u64;
-
- /// Sealed trait to start to remove specialisation from implementations
- ///
- /// This is done to force the associated value type to be unimplementable outside of this
- /// crate, and thus hint to the type system (and end user) traits are public for the contract
- /// and not for extension.
- pub trait ParquetValueType:
- std::cmp::PartialEq
- + std::fmt::Debug
- + std::fmt::Display
- + std::default::Default
- + std::clone::Clone
- + super::AsBytes
- + super::FromBytes
- + super::SliceAsBytes
- + PartialOrd
- {
- /// Encode the value directly from a higher level encoder
- fn encode<W: std::io::Write>(
- values: &[Self],
- writer: &mut W,
- bit_writer: &mut BitWriter,
- ) -> Result<()>;
-
- /// Establish the data that will be decoded in a buffer
- fn set_data(
- decoder: &mut PlainDecoderDetails,
- data: ByteBufferPtr,
- num_values: usize,
- );
-
- /// Decode the value from a given buffer for a higher level decoder
- fn decode(
- buffer: &mut [Self],
- decoder: &mut PlainDecoderDetails,
- ) -> Result<usize>;
-
- /// Return the encoded size for a type
- fn dict_encoding_size(&self) -> (usize, usize) {
- (std::mem::size_of::<Self>(), 1)
- }
-
- /// Return the value as i64 if possible
- ///
- /// This is essentially the same as `std::convert::TryInto<i64>` but can
- /// implemented for `f32` and `f64`, types that would fail orphan rules
- fn as_i64(&self) -> Result<i64> {
- Err(general_err!("Type cannot be converted to i64"))
- }
-
- /// Return the value as u64 if possible
- ///
- /// This is essentially the same as `std::convert::TryInto<u64>` but can
- /// implemented for `f32` and `f64`, types that would fail orphan rules
- fn as_u64(&self) -> Result<u64> {
- self.as_i64()
- .map_err(|_| general_err!("Type cannot be converted to u64"))
- .map(|x| x as u64)
- }
-
- /// Return the value as an Any to allow for downcasts without transmutation
- fn as_any(&self) -> &dyn std::any::Any;
-
- /// Return the value as an mutable Any to allow for downcasts without transmutation
- fn as_mut_any(&mut self) -> &mut dyn std::any::Any;
- }
-
- impl ParquetValueType for bool {
- #[inline]
- fn encode<W: std::io::Write>(
- values: &[Self],
- _: &mut W,
- bit_writer: &mut BitWriter,
- ) -> Result<()> {
- if bit_writer.bytes_written() + values.len() / 8 >= bit_writer.capacity() {
- bit_writer.extend(256);
- }
- for value in values {
- if !bit_writer.put_value(*value as u64, 1) {
- return Err(ParquetError::EOF(
- "unable to put boolean value".to_string(),
- ));
- }
- }
- Ok(())
- }
-
- #[inline]
- fn set_data(
- decoder: &mut PlainDecoderDetails,
- data: ByteBufferPtr,
- num_values: usize,
- ) {
- decoder.bit_reader.replace(BitReader::new(data));
- decoder.num_values = num_values;
- }
-
- #[inline]
- fn decode(
- buffer: &mut [Self],
- decoder: &mut PlainDecoderDetails,
- ) -> Result<usize> {
- let bit_reader = decoder.bit_reader.as_mut().unwrap();
- let num_values = std::cmp::min(buffer.len(), decoder.num_values);
- let values_read = bit_reader.get_batch(&mut buffer[..num_values], 1);
- decoder.num_values -= values_read;
- Ok(values_read)
- }
-
- #[inline]
- fn as_i64(&self) -> Result<i64> {
- Ok(*self as i64)
- }
-
- #[inline]
- fn as_any(&self) -> &dyn std::any::Any {
- self
- }
-
- #[inline]
- fn as_mut_any(&mut self) -> &mut dyn std::any::Any {
- self
- }
- }
-
- /// Hopelessly unsafe function that emulates `num::as_ne_bytes`
- ///
- /// It is not recommended to use this outside of this private module as, while it
- /// _should_ work for primitive values, it is little better than a transmutation
- /// and can act as a backdoor into mis-interpreting types as arbitary byte slices
- #[inline]
- fn as_raw<'a, T>(value: *const T) -> &'a [u8] {
- unsafe {
- let value = value as *const u8;
- std::slice::from_raw_parts(value, std::mem::size_of::<T>())
- }
- }
-
- macro_rules! impl_from_raw {
- ($ty: ty, $self: ident => $as_i64: block) => {
- impl ParquetValueType for $ty {
- #[inline]
- fn encode<W: std::io::Write>(values: &[Self], writer: &mut W, _: &mut BitWriter) -> Result<()> {
- let raw = unsafe {
- std::slice::from_raw_parts(
- values.as_ptr() as *const u8,
- std::mem::size_of::<$ty>() * values.len(),
- )
- };
- writer.write_all(raw)?;
-
- Ok(())
- }
-
- #[inline]
- fn set_data(decoder: &mut PlainDecoderDetails, data: ByteBufferPtr, num_values: usize) {
- decoder.data.replace(data);
- decoder.start = 0;
- decoder.num_values = num_values;
- }
-
- #[inline]
- fn decode(buffer: &mut [Self], decoder: &mut PlainDecoderDetails) -> Result<usize> {
- let data = decoder.data.as_ref().expect("set_data should have been called");
- let num_values = std::cmp::min(buffer.len(), decoder.num_values);
- let bytes_left = data.len() - decoder.start;
- let bytes_to_decode = std::mem::size_of::<Self>() * num_values;
-
- if bytes_left < bytes_to_decode {
- return Err(eof_err!("Not enough bytes to decode"));
- }
-
- // SAFETY: Raw types should be as per the standard rust bit-vectors
- unsafe {
- let raw_buffer = &mut Self::slice_as_bytes_mut(buffer)[..bytes_to_decode];
- raw_buffer.copy_from_slice(data.range(decoder.start, bytes_to_decode).as_ref());
- };
- decoder.start += bytes_to_decode;
- decoder.num_values -= num_values;
-
- Ok(num_values)
- }
-
- #[inline]
- fn as_i64(&$self) -> Result<i64> {
- $as_i64
- }
-
- #[inline]
- fn as_any(&self) -> &dyn std::any::Any {
- self
- }
-
- #[inline]
- fn as_mut_any(&mut self) -> &mut dyn std::any::Any {
- self
- }
- }
- }
- }
-
- impl_from_raw!(i32, self => { Ok(*self as i64) });
- impl_from_raw!(i64, self => { Ok(*self) });
- impl_from_raw!(f32, self => { Err(general_err!("Type cannot be converted to i64")) });
- impl_from_raw!(f64, self => { Err(general_err!("Type cannot be converted to i64")) });
-
- impl ParquetValueType for super::Int96 {
- #[inline]
- fn encode<W: std::io::Write>(
- values: &[Self],
- writer: &mut W,
- _: &mut BitWriter,
- ) -> Result<()> {
- for value in values {
- let raw = unsafe {
- std::slice::from_raw_parts(
- value.data() as *const [u32] as *const u8,
- 12,
- )
- };
- writer.write_all(raw)?;
- }
- Ok(())
- }
-
- #[inline]
- fn set_data(
- decoder: &mut PlainDecoderDetails,
- data: ByteBufferPtr,
- num_values: usize,
- ) {
- decoder.data.replace(data);
- decoder.start = 0;
- decoder.num_values = num_values;
- }
-
- #[inline]
- fn decode(
- buffer: &mut [Self],
- decoder: &mut PlainDecoderDetails,
- ) -> Result<usize> {
- // TODO - Remove the duplication between this and the general slice method
- let data = decoder
- .data
- .as_ref()
- .expect("set_data should have been called");
- let num_values = std::cmp::min(buffer.len(), decoder.num_values);
- let bytes_left = data.len() - decoder.start;
- let bytes_to_decode = 12 * num_values;
-
- if bytes_left < bytes_to_decode {
- return Err(eof_err!("Not enough bytes to decode"));
- }
-
- let data_range = data.range(decoder.start, bytes_to_decode);
- let bytes: &[u8] = data_range.data();
- decoder.start += bytes_to_decode;
-
- let mut pos = 0; // position in byte array
- for i in 0..num_values {
- let elem0 = byteorder::LittleEndian::read_u32(&bytes[pos..pos + 4]);
- let elem1 = byteorder::LittleEndian::read_u32(&bytes[pos + 4..pos + 8]);
- let elem2 = byteorder::LittleEndian::read_u32(&bytes[pos + 8..pos + 12]);
-
- buffer[i]
- .as_mut_any()
- .downcast_mut::<Self>()
- .unwrap()
- .set_data(elem0, elem1, elem2);
-
- pos += 12;
- }
- decoder.num_values -= num_values;
-
- Ok(num_values)
- }
-
- #[inline]
- fn as_any(&self) -> &dyn std::any::Any {
- self
- }
-
- #[inline]
- fn as_mut_any(&mut self) -> &mut dyn std::any::Any {
- self
- }
- }
-
- // TODO - Why does macro importing fail?
- /// Reads `$size` of bytes from `$src`, and reinterprets them as type `$ty`, in
- /// little-endian order. `$ty` must implement the `Default` trait. Otherwise this won't
- /// compile.
- /// This is copied and modified from byteorder crate.
- macro_rules! read_num_bytes {
- ($ty:ty, $size:expr, $src:expr) => {{
- assert!($size <= $src.len());
- let mut buffer =
- <$ty as $crate::util::bit_util::FromBytes>::Buffer::default();
- buffer.as_mut()[..$size].copy_from_slice(&$src[..$size]);
- <$ty>::from_ne_bytes(buffer)
- }};
- }
-
- impl ParquetValueType for super::ByteArray {
- #[inline]
- fn encode<W: std::io::Write>(
- values: &[Self],
- writer: &mut W,
- _: &mut BitWriter,
- ) -> Result<()> {
- for value in values {
- let len: u32 = value.len().try_into().unwrap();
- writer.write_all(&len.to_ne_bytes())?;
- let raw = value.data();
- writer.write_all(raw)?;
- }
- Ok(())
- }
-
- #[inline]
- fn set_data(
- decoder: &mut PlainDecoderDetails,
- data: ByteBufferPtr,
- num_values: usize,
- ) {
- decoder.data.replace(data);
- decoder.start = 0;
- decoder.num_values = num_values;
- }
-
- #[inline]
- fn decode(
- buffer: &mut [Self],
- decoder: &mut PlainDecoderDetails,
- ) -> Result<usize> {
- let data = decoder
- .data
- .as_mut()
- .expect("set_data should have been called");
- let num_values = std::cmp::min(buffer.len(), decoder.num_values);
- for i in 0..num_values {
- let len: usize =
- read_num_bytes!(u32, 4, data.start_from(decoder.start).as_ref())
- as usize;
- decoder.start += std::mem::size_of::<u32>();
-
- if data.len() < decoder.start + len {
- return Err(eof_err!("Not enough bytes to decode"));
- }
-
- let val: &mut Self = buffer[i].as_mut_any().downcast_mut().unwrap();
-
- val.set_data(data.range(decoder.start, len));
- decoder.start += len;
- }
- decoder.num_values -= num_values;
-
- Ok(num_values)
- }
-
- #[inline]
- fn dict_encoding_size(&self) -> (usize, usize) {
- (std::mem::size_of::<u32>(), self.len())
- }
-
- #[inline]
- fn as_any(&self) -> &dyn std::any::Any {
- self
- }
-
- #[inline]
- fn as_mut_any(&mut self) -> &mut dyn std::any::Any {
- self
- }
- }
-
- impl ParquetValueType for super::FixedLenByteArray {
- #[inline]
- fn encode<W: std::io::Write>(
- values: &[Self],
- writer: &mut W,
- _: &mut BitWriter,
- ) -> Result<()> {
- for value in values {
- let raw = value.data();
- writer.write_all(raw)?;
- }
- Ok(())
- }
-
- #[inline]
- fn set_data(
- decoder: &mut PlainDecoderDetails,
- data: ByteBufferPtr,
- num_values: usize,
- ) {
- decoder.data.replace(data);
- decoder.start = 0;
- decoder.num_values = num_values;
- }
-
- #[inline]
- fn decode(
- buffer: &mut [Self],
- decoder: &mut PlainDecoderDetails,
- ) -> Result<usize> {
- assert!(decoder.type_length > 0);
-
- let data = decoder
- .data
- .as_mut()
- .expect("set_data should have been called");
- let num_values = std::cmp::min(buffer.len(), decoder.num_values);
- for i in 0..num_values {
- let len = decoder.type_length as usize;
-
- if data.len() < decoder.start + len {
- return Err(eof_err!("Not enough bytes to decode"));
- }
-
- let val: &mut Self = buffer[i].as_mut_any().downcast_mut().unwrap();
-
- val.set_data(data.range(decoder.start, len));
- decoder.start += len;
- }
- decoder.num_values -= num_values;
-
- Ok(num_values)
- }
-
- #[inline]
- fn dict_encoding_size(&self) -> (usize, usize) {
- (std::mem::size_of::<u32>(), self.len())
- }
-
- #[inline]
- fn as_any(&self) -> &dyn std::any::Any {
- self
- }
-
- #[inline]
- fn as_mut_any(&mut self) -> &mut dyn std::any::Any {
- self
- }
- }
-}
-
-/// Contains the Parquet physical type information as well as the Rust primitive type
-/// presentation.
-pub trait DataType: 'static {
- type T: private::ParquetValueType;
-
- /// Returns Parquet physical type.
- fn get_physical_type() -> Type;
-
- /// Returns size in bytes for Rust representation of the physical type.
- fn get_type_size() -> usize;
-
- fn get_column_reader(column_writer: ColumnReader) -> Option<ColumnReaderImpl<Self>>
- where
- Self: Sized;
-
- fn get_column_writer(column_writer: ColumnWriter) -> Option<ColumnWriterImpl<Self>>
- where
- Self: Sized;
-
- fn get_column_writer_ref(
- column_writer: &ColumnWriter,
- ) -> Option<&ColumnWriterImpl<Self>>
- where
- Self: Sized;
-
- fn get_column_writer_mut(
- column_writer: &mut ColumnWriter,
- ) -> Option<&mut ColumnWriterImpl<Self>>
- where
- Self: Sized;
-}
-
-// Workaround bug in specialization
-pub trait SliceAsBytesDataType: DataType
-where
- Self::T: SliceAsBytes,
-{
-}
-
-impl<T> SliceAsBytesDataType for T
-where
- T: DataType,
- <T as DataType>::T: SliceAsBytes,
-{
-}
-
-macro_rules! make_type {
- ($name:ident, $physical_ty:path, $reader_ident: ident, $writer_ident: ident, $native_ty:ty, $size:expr) => {
- #[derive(Clone)]
- pub struct $name {}
-
- impl DataType for $name {
- type T = $native_ty;
-
- fn get_physical_type() -> Type {
- $physical_ty
- }
-
- fn get_type_size() -> usize {
- $size
- }
-
- fn get_column_reader(
- column_writer: ColumnReader,
- ) -> Option<ColumnReaderImpl<Self>> {
- match column_writer {
- ColumnReader::$reader_ident(w) => Some(w),
- _ => None,
- }
- }
-
- fn get_column_writer(
- column_writer: ColumnWriter,
- ) -> Option<ColumnWriterImpl<Self>> {
- match column_writer {
- ColumnWriter::$writer_ident(w) => Some(w),
- _ => None,
- }
- }
-
- fn get_column_writer_ref(
- column_writer: &ColumnWriter,
- ) -> Option<&ColumnWriterImpl<Self>> {
- match column_writer {
- ColumnWriter::$writer_ident(w) => Some(w),
- _ => None,
- }
- }
-
- fn get_column_writer_mut(
- column_writer: &mut ColumnWriter,
- ) -> Option<&mut ColumnWriterImpl<Self>> {
- match column_writer {
- ColumnWriter::$writer_ident(w) => Some(w),
- _ => None,
- }
- }
- }
- };
-}
-
-// Generate struct definitions for all physical types
-
-make_type!(
- BoolType,
- Type::BOOLEAN,
- BoolColumnReader,
- BoolColumnWriter,
- bool,
- 1
-);
-make_type!(
- Int32Type,
- Type::INT32,
- Int32ColumnReader,
- Int32ColumnWriter,
- i32,
- 4
-);
-make_type!(
- Int64Type,
- Type::INT64,
- Int64ColumnReader,
- Int64ColumnWriter,
- i64,
- 8
-);
-make_type!(
- Int96Type,
- Type::INT96,
- Int96ColumnReader,
- Int96ColumnWriter,
- Int96,
- mem::size_of::<Int96>()
-);
-make_type!(
- FloatType,
- Type::FLOAT,
- FloatColumnReader,
- FloatColumnWriter,
- f32,
- 4
-);
-make_type!(
- DoubleType,
- Type::DOUBLE,
- DoubleColumnReader,
- DoubleColumnWriter,
- f64,
- 8
-);
-make_type!(
- ByteArrayType,
- Type::BYTE_ARRAY,
- ByteArrayColumnReader,
- ByteArrayColumnWriter,
- ByteArray,
- mem::size_of::<ByteArray>()
-);
-make_type!(
- FixedLenByteArrayType,
- Type::FIXED_LEN_BYTE_ARRAY,
- FixedLenByteArrayColumnReader,
- FixedLenByteArrayColumnWriter,
- FixedLenByteArray,
- mem::size_of::<FixedLenByteArray>()
-);
-
-impl FromBytes for Int96 {
- type Buffer = [u8; 12];
- fn from_le_bytes(_bs: Self::Buffer) -> Self {
- unimplemented!()
- }
- fn from_be_bytes(_bs: Self::Buffer) -> Self {
- unimplemented!()
- }
- fn from_ne_bytes(bs: Self::Buffer) -> Self {
- let mut i = Int96::new();
- i.set_data(
- from_ne_slice(&bs[0..4]),
- from_ne_slice(&bs[4..8]),
- from_ne_slice(&bs[8..12]),
- );
- i
- }
-}
-
-// FIXME Needed to satisfy the constraint of many decoding functions but ByteArray does not
-// appear to actual be converted directly from bytes
-impl FromBytes for ByteArray {
- type Buffer = [u8; 8];
- fn from_le_bytes(_bs: Self::Buffer) -> Self {
- unreachable!()
- }
- fn from_be_bytes(_bs: Self::Buffer) -> Self {
- unreachable!()
- }
- fn from_ne_bytes(bs: Self::Buffer) -> Self {
- ByteArray::from(bs.to_vec())
- }
-}
-
-impl FromBytes for FixedLenByteArray {
- type Buffer = [u8; 8];
-
- fn from_le_bytes(_bs: Self::Buffer) -> Self {
- unreachable!()
- }
- fn from_be_bytes(_bs: Self::Buffer) -> Self {
- unreachable!()
- }
- fn from_ne_bytes(bs: Self::Buffer) -> Self {
- Self(ByteArray::from(bs.to_vec()))
- }
-}
-
-/// Macro to reduce repetition in making type assertions on the physical type against `T`
-macro_rules! ensure_phys_ty {
- ($($ty: pat)|+ , $err: literal) => {
- match T::get_physical_type() {
- $($ty => (),)*
- _ => panic!($err),
- };
- }
-}
-
-#[cfg(test)]
-#[allow(clippy::float_cmp, clippy::approx_constant)]
-mod tests {
- use super::*;
-
- #[test]
- #[allow(clippy::string_lit_as_bytes)]
- fn test_as_bytes() {
- assert_eq!(false.as_bytes(), &[0]);
- assert_eq!(true.as_bytes(), &[1]);
- assert_eq!(7_i32.as_bytes(), &[7, 0, 0, 0]);
- assert_eq!(555_i32.as_bytes(), &[43, 2, 0, 0]);
- assert_eq!(555_u32.as_bytes(), &[43, 2, 0, 0]);
- assert_eq!(i32::max_value().as_bytes(), &[255, 255, 255, 127]);
- assert_eq!(i32::min_value().as_bytes(), &[0, 0, 0, 128]);
- assert_eq!(7_i64.as_bytes(), &[7, 0, 0, 0, 0, 0, 0, 0]);
- assert_eq!(555_i64.as_bytes(), &[43, 2, 0, 0, 0, 0, 0, 0]);
- assert_eq!(
- (i64::max_value()).as_bytes(),
- &[255, 255, 255, 255, 255, 255, 255, 127]
- );
- assert_eq!((i64::min_value()).as_bytes(), &[0, 0, 0, 0, 0, 0, 0, 128]);
- assert_eq!(3.14_f32.as_bytes(), &[195, 245, 72, 64]);
- assert_eq!(3.14_f64.as_bytes(), &[31, 133, 235, 81, 184, 30, 9, 64]);
- assert_eq!("hello".as_bytes(), &[b'h', b'e', b'l', b'l', b'o']);
- assert_eq!(
- Vec::from("hello".as_bytes()).as_bytes(),
- &[b'h', b'e', b'l', b'l', b'o']
- );
-
- // Test Int96
- let i96 = Int96::from(vec![1, 2, 3]);
- assert_eq!(i96.as_bytes(), &[1, 0, 0, 0, 2, 0, 0, 0, 3, 0, 0, 0]);
-
- // Test ByteArray
- let ba = ByteArray::from(vec![1, 2, 3]);
- assert_eq!(ba.as_bytes(), &[1, 2, 3]);
-
- // Test Decimal
- let decimal = Decimal::from_i32(123, 5, 2);
- assert_eq!(decimal.as_bytes(), &[0, 0, 0, 123]);
- let decimal = Decimal::from_i64(123, 5, 2);
- assert_eq!(decimal.as_bytes(), &[0, 0, 0, 0, 0, 0, 0, 123]);
- let decimal = Decimal::from_bytes(ByteArray::from(vec![1, 2, 3]), 5, 2);
- assert_eq!(decimal.as_bytes(), &[1, 2, 3]);
- }
-
- #[test]
- fn test_int96_from() {
- assert_eq!(
- Int96::from(vec![1, 12345, 1234567890]).data(),
- &[1, 12345, 1234567890]
- );
- }
-
- #[test]
- fn test_byte_array_from() {
- assert_eq!(
- ByteArray::from(vec![b'A', b'B', b'C']).data(),
- &[b'A', b'B', b'C']
- );
- assert_eq!(ByteArray::from("ABC").data(), &[b'A', b'B', b'C']);
- assert_eq!(
- ByteArray::from(ByteBufferPtr::new(vec![1u8, 2u8, 3u8, 4u8, 5u8])).data(),
- &[1u8, 2u8, 3u8, 4u8, 5u8]
- );
- let mut buf = ByteBuffer::new();
- buf.set_data(vec![6u8, 7u8, 8u8, 9u8, 10u8]);
- assert_eq!(ByteArray::from(buf).data(), &[6u8, 7u8, 8u8, 9u8, 10u8]);
- }
-
- #[test]
- fn test_decimal_partial_eq() {
- assert_eq!(Decimal::default(), Decimal::from_i32(0, 0, 0));
- assert_eq!(Decimal::from_i32(222, 5, 2), Decimal::from_i32(222, 5, 2));
- assert_eq!(
- Decimal::from_bytes(ByteArray::from(vec![0, 0, 0, 3]), 5, 2),
- Decimal::from_i32(3, 5, 2)
- );
-
- assert!(Decimal::from_i32(222, 5, 2) != Decimal::from_i32(111, 5, 2));
- assert!(Decimal::from_i32(222, 5, 2) != Decimal::from_i32(222, 6, 2));
- assert!(Decimal::from_i32(222, 5, 2) != Decimal::from_i32(222, 5, 3));
-
- assert!(Decimal::from_i64(222, 5, 2) != Decimal::from_i32(222, 5, 2));
- }
-
- #[test]
- fn test_byte_array_ord() {
- let ba1 = ByteArray::from(vec![1, 2, 3]);
- let ba11 = ByteArray::from(vec![1, 2, 3]);
- let ba2 = ByteArray::from(vec![3, 4]);
- let ba3 = ByteArray::from(vec![1, 2, 4]);
- let ba4 = ByteArray::from(vec![]);
- let ba5 = ByteArray::from(vec![2, 2, 3]);
-
- assert!(ba1 > ba2);
- assert!(ba3 > ba1);
- assert!(ba1 > ba4);
- assert_eq!(ba1, ba11);
- assert!(ba5 > ba1);
- }
-}
diff --git a/parquet/src/encodings/decoding.rs b/parquet/src/encodings/decoding.rs
deleted file mode 100644
index e83e277..0000000
--- a/parquet/src/encodings/decoding.rs
+++ /dev/null
@@ -1,1386 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-//! Contains all supported decoders for Parquet.
-
-use std::{cmp, marker::PhantomData, mem};
-
-use super::rle::RleDecoder;
-
-use crate::basic::*;
-use crate::data_type::private::*;
-use crate::data_type::*;
-use crate::errors::{ParquetError, Result};
-use crate::schema::types::ColumnDescPtr;
-use crate::util::{
- bit_util::{self, BitReader, FromBytes},
- memory::{ByteBuffer, ByteBufferPtr},
-};
-
-// ----------------------------------------------------------------------
-// Decoders
-
-/// A Parquet decoder for the data type `T`.
-pub trait Decoder<T: DataType> {
- /// Sets the data to decode to be `data`, which should contain `num_values` of values
- /// to decode.
- fn set_data(&mut self, data: ByteBufferPtr, num_values: usize) -> Result<()>;
-
- /// Consumes values from this decoder and write the results to `buffer`. This will try
- /// to fill up `buffer`.
- ///
- /// Returns the actual number of values decoded, which should be equal to
- /// `buffer.len()` unless the remaining number of values is less than
- /// `buffer.len()`.
- fn get(&mut self, buffer: &mut [T::T]) -> Result<usize>;
-
- /// Consume values from this decoder and write the results to `buffer`, leaving
- /// "spaces" for null values.
- ///
- /// `null_count` is the number of nulls we expect to see in `buffer`, after reading.
- /// `valid_bits` stores the valid bit for each value in the buffer. It should contain
- /// at least number of bits that equal to `buffer.len()`.
- ///
- /// Returns the actual number of values decoded.
- ///
- /// # Panics
- ///
- /// Panics if `null_count` is greater than `buffer.len()`.
- fn get_spaced(
- &mut self,
- buffer: &mut [T::T],
- null_count: usize,
- valid_bits: &[u8],
- ) -> Result<usize> {
- assert!(buffer.len() >= null_count);
-
- // TODO: check validity of the input arguments?
- if null_count == 0 {
- return self.get(buffer);
- }
-
- let num_values = buffer.len();
- let values_to_read = num_values - null_count;
- let values_read = self.get(buffer)?;
- if values_read != values_to_read {
- return Err(general_err!(
- "Number of values read: {}, doesn't match expected: {}",
- values_read,
- values_to_read
- ));
- }
- let mut values_to_move = values_read;
- for i in (0..num_values).rev() {
- if bit_util::get_bit(valid_bits, i) {
- values_to_move -= 1;
- buffer.swap(i, values_to_move);
- }
- }
-
- Ok(num_values)
- }
-
- /// Returns the number of values left in this decoder stream.
- fn values_left(&self) -> usize;
-
- /// Returns the encoding for this decoder.
- fn encoding(&self) -> Encoding;
-}
-
-/// Gets a decoder for the column descriptor `descr` and encoding type `encoding`.
-///
-/// NOTE: the primitive type in `descr` MUST match the data type `T`, otherwise
-/// disastrous consequence could occur.
-pub fn get_decoder<T: DataType>(
- descr: ColumnDescPtr,
- encoding: Encoding,
-) -> Result<Box<dyn Decoder<T>>> {
- let decoder: Box<dyn Decoder<T>> = match encoding {
- Encoding::PLAIN => Box::new(PlainDecoder::new(descr.type_length())),
- Encoding::RLE_DICTIONARY | Encoding::PLAIN_DICTIONARY => {
- return Err(general_err!(
- "Cannot initialize this encoding through this function"
- ));
- }
- Encoding::RLE => Box::new(RleValueDecoder::new()),
- Encoding::DELTA_BINARY_PACKED => Box::new(DeltaBitPackDecoder::new()),
- Encoding::DELTA_LENGTH_BYTE_ARRAY => Box::new(DeltaLengthByteArrayDecoder::new()),
- Encoding::DELTA_BYTE_ARRAY => Box::new(DeltaByteArrayDecoder::new()),
- e => return Err(nyi_err!("Encoding {} is not supported", e)),
- };
- Ok(decoder)
-}
-
-// ----------------------------------------------------------------------
-// PLAIN Decoding
-
-#[derive(Default)]
-pub struct PlainDecoderDetails {
- // The remaining number of values in the byte array
- pub(crate) num_values: usize,
-
- // The current starting index in the byte array. Not used when `T` is bool.
- pub(crate) start: usize,
-
- // The length for the type `T`. Only used when `T` is `FixedLenByteArrayType`
- pub(crate) type_length: i32,
-
- // The byte array to decode from. Not set if `T` is bool.
- pub(crate) data: Option<ByteBufferPtr>,
-
- // Read `data` bit by bit. Only set if `T` is bool.
- pub(crate) bit_reader: Option<BitReader>,
-}
-
-/// Plain decoding that supports all types.
-/// Values are encoded back to back. For native types, data is encoded as little endian.
-/// Floating point types are encoded in IEEE.
-/// See [`PlainEncoder`](crate::encoding::PlainEncoder) for more information.
-pub struct PlainDecoder<T: DataType> {
- // The binary details needed for decoding
- inner: PlainDecoderDetails,
-
- // To allow `T` in the generic parameter for this struct. This doesn't take any
- // space.
- _phantom: PhantomData<T>,
-}
-
-impl<T: DataType> PlainDecoder<T> {
- /// Creates new plain decoder.
- pub fn new(type_length: i32) -> Self {
- PlainDecoder {
- inner: PlainDecoderDetails {
- type_length,
- num_values: 0,
- start: 0,
- data: None,
- bit_reader: None,
- },
- _phantom: PhantomData,
- }
- }
-}
-
-impl<T: DataType> Decoder<T> for PlainDecoder<T> {
- #[inline]
- fn set_data(&mut self, data: ByteBufferPtr, num_values: usize) -> Result<()> {
- T::T::set_data(&mut self.inner, data, num_values);
- Ok(())
- }
-
- #[inline]
- fn values_left(&self) -> usize {
- self.inner.num_values
- }
-
- #[inline]
- fn encoding(&self) -> Encoding {
- Encoding::PLAIN
- }
-
- #[inline]
- fn get(&mut self, buffer: &mut [T::T]) -> Result<usize> {
- T::T::decode(buffer, &mut self.inner)
- }
-}
-
-// ----------------------------------------------------------------------
-// RLE_DICTIONARY/PLAIN_DICTIONARY Decoding
-
-/// Dictionary decoder.
-/// The dictionary encoding builds a dictionary of values encountered in a given column.
-/// The dictionary is be stored in a dictionary page per column chunk.
-/// See [`DictEncoder`](crate::encoding::DictEncoder) for more information.
-pub struct DictDecoder<T: DataType> {
- // The dictionary, which maps ids to the values
- dictionary: Vec<T::T>,
-
- // Whether `dictionary` has been initialized
- has_dictionary: bool,
-
- // The decoder for the value ids
- rle_decoder: Option<RleDecoder>,
-
- // Number of values left in the data stream
- num_values: usize,
-}
-
-impl<T: DataType> DictDecoder<T> {
- /// Creates new dictionary decoder.
- pub fn new() -> Self {
- Self {
- dictionary: vec![],
- has_dictionary: false,
- rle_decoder: None,
- num_values: 0,
- }
- }
-
- /// Decodes and sets values for dictionary using `decoder` decoder.
- pub fn set_dict(&mut self, mut decoder: Box<dyn Decoder<T>>) -> Result<()> {
- let num_values = decoder.values_left();
- self.dictionary.resize(num_values, T::T::default());
- let _ = decoder.get(&mut self.dictionary)?;
- self.has_dictionary = true;
- Ok(())
- }
-}
-
-impl<T: DataType> Decoder<T> for DictDecoder<T> {
- fn set_data(&mut self, data: ByteBufferPtr, num_values: usize) -> Result<()> {
- // First byte in `data` is bit width
- let bit_width = data.as_ref()[0];
- let mut rle_decoder = RleDecoder::new(bit_width);
- rle_decoder.set_data(data.start_from(1));
- self.num_values = num_values;
- self.rle_decoder = Some(rle_decoder);
- Ok(())
- }
-
- fn get(&mut self, buffer: &mut [T::T]) -> Result<usize> {
- assert!(self.rle_decoder.is_some());
- assert!(self.has_dictionary, "Must call set_dict() first!");
-
- let rle = self.rle_decoder.as_mut().unwrap();
- let num_values = cmp::min(buffer.len(), self.num_values);
- rle.get_batch_with_dict(&self.dictionary[..], buffer, num_values)
- }
-
- /// Number of values left in this decoder stream
- fn values_left(&self) -> usize {
- self.num_values
- }
-
- fn encoding(&self) -> Encoding {
- Encoding::RLE_DICTIONARY
- }
-}
-
-// ----------------------------------------------------------------------
-// RLE Decoding
-
-/// RLE/Bit-Packing hybrid decoding for values.
-/// Currently is used only for data pages v2 and supports boolean types.
-/// See [`RleValueEncoder`](crate::encoding::RleValueEncoder) for more information.
-pub struct RleValueDecoder<T: DataType> {
- values_left: usize,
- decoder: RleDecoder,
- _phantom: PhantomData<T>,
-}
-
-impl<T: DataType> RleValueDecoder<T> {
- pub fn new() -> Self {
- Self {
- values_left: 0,
- decoder: RleDecoder::new(1),
- _phantom: PhantomData,
- }
- }
-}
-
-impl<T: DataType> Decoder<T> for RleValueDecoder<T> {
- #[inline]
- fn set_data(&mut self, data: ByteBufferPtr, num_values: usize) -> Result<()> {
- // Only support RLE value reader for boolean values with bit width of 1.
- ensure_phys_ty!(Type::BOOLEAN, "RleValueDecoder only supports BoolType");
-
- // We still need to remove prefix of i32 from the stream.
- const I32_SIZE: usize = mem::size_of::<i32>();
- let data_size = read_num_bytes!(i32, I32_SIZE, data.as_ref()) as usize;
- self.decoder = RleDecoder::new(1);
- self.decoder.set_data(data.range(I32_SIZE, data_size));
- self.values_left = num_values;
- Ok(())
- }
-
- #[inline]
- fn values_left(&self) -> usize {
- self.values_left
- }
-
- #[inline]
- fn encoding(&self) -> Encoding {
- Encoding::RLE
- }
-
- #[inline]
- fn get(&mut self, buffer: &mut [T::T]) -> Result<usize> {
- let num_values = cmp::min(buffer.len(), self.values_left);
- let values_read = self.decoder.get_batch(&mut buffer[..num_values])?;
- self.values_left -= values_read;
- Ok(values_read)
- }
-}
-
-// ----------------------------------------------------------------------
-// DELTA_BINARY_PACKED Decoding
-
-/// Delta binary packed decoder.
-/// Supports INT32 and INT64 types.
-/// See [`DeltaBitPackEncoder`](crate::encoding::DeltaBitPackEncoder) for more
-/// information.
-pub struct DeltaBitPackDecoder<T: DataType> {
- bit_reader: BitReader,
- initialized: bool,
-
- // Header info
- num_values: usize,
- num_mini_blocks: i64,
- values_per_mini_block: usize,
- values_current_mini_block: usize,
- first_value: i64,
- first_value_read: bool,
-
- // Per block info
- min_delta: i64,
- mini_block_idx: usize,
- delta_bit_width: u8,
- delta_bit_widths: ByteBuffer,
- deltas_in_mini_block: Vec<T::T>, // eagerly loaded deltas for a mini block
- use_batch: bool,
-
- current_value: i64,
-
- _phantom: PhantomData<T>,
-}
-
-impl<T: DataType> DeltaBitPackDecoder<T> {
- /// Creates new delta bit packed decoder.
- pub fn new() -> Self {
- Self {
- bit_reader: BitReader::from(vec![]),
- initialized: false,
- num_values: 0,
- num_mini_blocks: 0,
- values_per_mini_block: 0,
- values_current_mini_block: 0,
- first_value: 0,
- first_value_read: false,
- min_delta: 0,
- mini_block_idx: 0,
- delta_bit_width: 0,
- delta_bit_widths: ByteBuffer::new(),
- deltas_in_mini_block: vec![],
- use_batch: mem::size_of::<T::T>() == 4,
- current_value: 0,
- _phantom: PhantomData,
- }
- }
-
- /// Returns underlying bit reader offset.
- pub fn get_offset(&self) -> usize {
- assert!(self.initialized, "Bit reader is not initialized");
- self.bit_reader.get_byte_offset()
- }
-
- /// Initializes new mini block.
- #[inline]
- fn init_block(&mut self) -> Result<()> {
- self.min_delta = self
- .bit_reader
- .get_zigzag_vlq_int()
- .ok_or_else(|| eof_err!("Not enough data to decode 'min_delta'"))?;
-
- self.delta_bit_widths.clear();
- for _ in 0..self.num_mini_blocks {
- let w = self
- .bit_reader
- .get_aligned::<u8>(1)
- .ok_or_else(|| eof_err!("Not enough data to decode 'width'"))?;
- self.delta_bit_widths.push(w);
- }
-
- self.mini_block_idx = 0;
- self.delta_bit_width = self.delta_bit_widths.data()[0];
- self.values_current_mini_block = self.values_per_mini_block;
- Ok(())
- }
-
- /// Loads delta into mini block.
- #[inline]
- fn load_deltas_in_mini_block(&mut self) -> Result<()>
- where
- T::T: FromBytes,
- {
- if self.use_batch {
- self.deltas_in_mini_block
- .resize(self.values_current_mini_block, T::T::default());
- let loaded = self.bit_reader.get_batch::<T::T>(
- &mut self.deltas_in_mini_block[..],
- self.delta_bit_width as usize,
- );
- assert!(loaded == self.values_current_mini_block);
- } else {
- self.deltas_in_mini_block.clear();
- for _ in 0..self.values_current_mini_block {
- // TODO: load one batch at a time similar to int32
- let delta = self
- .bit_reader
- .get_value::<T::T>(self.delta_bit_width as usize)
- .ok_or_else(|| eof_err!("Not enough data to decode 'delta'"))?;
- self.deltas_in_mini_block.push(delta);
- }
- }
-
- Ok(())
- }
-}
-
-impl<T: DataType> Decoder<T> for DeltaBitPackDecoder<T> {
- // # of total values is derived from encoding
- #[inline]
- fn set_data(&mut self, data: ByteBufferPtr, _index: usize) -> Result<()> {
- self.bit_reader = BitReader::new(data);
- self.initialized = true;
-
- let block_size = self
- .bit_reader
- .get_vlq_int()
- .ok_or_else(|| eof_err!("Not enough data to decode 'block_size'"))?;
- self.num_mini_blocks = self
- .bit_reader
- .get_vlq_int()
- .ok_or_else(|| eof_err!("Not enough data to decode 'num_mini_blocks'"))?;
- self.num_values = self
- .bit_reader
- .get_vlq_int()
- .ok_or_else(|| eof_err!("Not enough data to decode 'num_values'"))?
- as usize;
- self.first_value = self
- .bit_reader
- .get_zigzag_vlq_int()
- .ok_or_else(|| eof_err!("Not enough data to decode 'first_value'"))?;
-
- // Reset decoding state
- self.first_value_read = false;
- self.mini_block_idx = 0;
- self.delta_bit_widths.clear();
- self.values_current_mini_block = 0;
-
- self.values_per_mini_block = (block_size / self.num_mini_blocks) as usize;
- assert!(self.values_per_mini_block % 8 == 0);
-
- Ok(())
- }
-
- fn get(&mut self, buffer: &mut [T::T]) -> Result<usize> {
- assert!(self.initialized, "Bit reader is not initialized");
-
- let num_values = cmp::min(buffer.len(), self.num_values);
- for i in 0..num_values {
- if !self.first_value_read {
- self.set_decoded_value(buffer, i, self.first_value);
- self.current_value = self.first_value;
- self.first_value_read = true;
- continue;
- }
-
- if self.values_current_mini_block == 0 {
- self.mini_block_idx += 1;
- if self.mini_block_idx < self.delta_bit_widths.size() {
- self.delta_bit_width =
- self.delta_bit_widths.data()[self.mini_block_idx];
- self.values_current_mini_block = self.values_per_mini_block;
- } else {
- self.init_block()?;
- }
- self.load_deltas_in_mini_block()?;
- }
-
- // we decrement values in current mini block, so we need to invert index for
- // delta
- let delta = self.get_delta(
- self.deltas_in_mini_block.len() - self.values_current_mini_block,
- );
- // It is OK for deltas to contain "overflowed" values after encoding,
- // e.g. i64::MAX - i64::MIN, so we use `wrapping_add` to "overflow" again and
- // restore original value.
- self.current_value = self.current_value.wrapping_add(self.min_delta);
- self.current_value = self.current_value.wrapping_add(delta as i64);
- self.set_decoded_value(buffer, i, self.current_value);
- self.values_current_mini_block -= 1;
- }
-
- self.num_values -= num_values;
- Ok(num_values)
- }
-
- fn values_left(&self) -> usize {
- self.num_values
- }
-
- fn encoding(&self) -> Encoding {
- Encoding::DELTA_BINARY_PACKED
- }
-}
-
-/// Helper trait to define specific conversions when decoding values
-trait DeltaBitPackDecoderConversion<T: DataType> {
- /// Sets decoded value based on type `T`.
- fn get_delta(&self, index: usize) -> i64;
-
- fn set_decoded_value(&self, buffer: &mut [T::T], index: usize, value: i64);
-}
-
-impl<T: DataType> DeltaBitPackDecoderConversion<T> for DeltaBitPackDecoder<T> {
- #[inline]
- fn get_delta(&self, index: usize) -> i64 {
- ensure_phys_ty!(
- Type::INT32 | Type::INT64,
- "DeltaBitPackDecoder only supports Int32Type and Int64Type"
- );
- self.deltas_in_mini_block[index].as_i64().unwrap()
- }
-
- #[inline]
- fn set_decoded_value(&self, buffer: &mut [T::T], index: usize, value: i64) {
- match T::get_physical_type() {
- Type::INT32 => {
- let val = buffer[index].as_mut_any().downcast_mut::<i32>().unwrap();
-
- *val = value as i32;
- }
- Type::INT64 => {
- let val = buffer[index].as_mut_any().downcast_mut::<i64>().unwrap();
-
- *val = value;
- }
- _ => panic!("DeltaBitPackDecoder only supports Int32Type and Int64Type"),
- };
- }
-}
-
-// ----------------------------------------------------------------------
-// DELTA_LENGTH_BYTE_ARRAY Decoding
-
-/// Delta length byte array decoder.
-/// Only applied to byte arrays to separate the length values and the data, the lengths
-/// are encoded using DELTA_BINARY_PACKED encoding.
-/// See [`DeltaLengthByteArrayEncoder`](crate::encoding::DeltaLengthByteArrayEncoder)
-/// for more information.
-pub struct DeltaLengthByteArrayDecoder<T: DataType> {
- // Lengths for each byte array in `data`
- // TODO: add memory tracker to this
- lengths: Vec<i32>,
-
- // Current index into `lengths`
- current_idx: usize,
-
- // Concatenated byte array data
- data: Option<ByteBufferPtr>,
-
- // Offset into `data`, always point to the beginning of next byte array.
- offset: usize,
-
- // Number of values left in this decoder stream
- num_values: usize,
-
- // Placeholder to allow `T` as generic parameter
- _phantom: PhantomData<T>,
-}
-
-impl<T: DataType> DeltaLengthByteArrayDecoder<T> {
- /// Creates new delta length byte array decoder.
- pub fn new() -> Self {
- Self {
- lengths: vec![],
- current_idx: 0,
- data: None,
- offset: 0,
- num_values: 0,
- _phantom: PhantomData,
- }
- }
-}
-
-impl<T: DataType> Decoder<T> for DeltaLengthByteArrayDecoder<T> {
- fn set_data(&mut self, data: ByteBufferPtr, num_values: usize) -> Result<()> {
- match T::get_physical_type() {
- Type::BYTE_ARRAY => {
- let mut len_decoder = DeltaBitPackDecoder::<Int32Type>::new();
- len_decoder.set_data(data.all(), num_values)?;
- let num_lengths = len_decoder.values_left();
- self.lengths.resize(num_lengths, 0);
- len_decoder.get(&mut self.lengths[..])?;
-
- self.data = Some(data.start_from(len_decoder.get_offset()));
- self.offset = 0;
- self.current_idx = 0;
- self.num_values = num_lengths;
- Ok(())
- }
- _ => Err(general_err!(
- "DeltaLengthByteArrayDecoder only support ByteArrayType"
- )),
- }
- }
-
- fn get(&mut self, buffer: &mut [T::T]) -> Result<usize> {
- match T::get_physical_type() {
- Type::BYTE_ARRAY => {
- assert!(self.data.is_some());
-
- let data = self.data.as_ref().unwrap();
- let num_values = cmp::min(buffer.len(), self.num_values);
- for i in 0..num_values {
- let len = self.lengths[self.current_idx] as usize;
-
- buffer[i]
- .as_mut_any()
- .downcast_mut::<ByteArray>()
- .unwrap()
- .set_data(data.range(self.offset, len));
-
- self.offset += len;
- self.current_idx += 1;
- }
-
- self.num_values -= num_values;
- Ok(num_values)
- }
- _ => Err(general_err!(
- "DeltaLengthByteArrayDecoder only support ByteArrayType"
- )),
- }
- }
-
- fn values_left(&self) -> usize {
- self.num_values
- }
-
- fn encoding(&self) -> Encoding {
- Encoding::DELTA_LENGTH_BYTE_ARRAY
- }
-}
-
-// ----------------------------------------------------------------------
-// DELTA_BYTE_ARRAY Decoding
-
-/// Delta byte array decoder.
-/// Prefix lengths are encoded using `DELTA_BINARY_PACKED` encoding, Suffixes are stored
-/// using `DELTA_LENGTH_BYTE_ARRAY` encoding.
-/// See [`DeltaByteArrayEncoder`](crate::encoding::DeltaByteArrayEncoder) for more
-/// information.
-pub struct DeltaByteArrayDecoder<T: DataType> {
- // Prefix lengths for each byte array
- // TODO: add memory tracker to this
- prefix_lengths: Vec<i32>,
-
- // The current index into `prefix_lengths`,
- current_idx: usize,
-
- // Decoder for all suffixes, the # of which should be the same as
- // `prefix_lengths.len()`
- suffix_decoder: Option<DeltaLengthByteArrayDecoder<ByteArrayType>>,
-
- // The last byte array, used to derive the current prefix
- previous_value: Vec<u8>,
-
- // Number of values left
- num_values: usize,
-
- // Placeholder to allow `T` as generic parameter
- _phantom: PhantomData<T>,
-}
-
-impl<T: DataType> DeltaByteArrayDecoder<T> {
- /// Creates new delta byte array decoder.
- pub fn new() -> Self {
- Self {
- prefix_lengths: vec![],
- current_idx: 0,
- suffix_decoder: None,
- previous_value: vec![],
- num_values: 0,
- _phantom: PhantomData,
- }
- }
-}
-
-impl<'m, T: DataType> Decoder<T> for DeltaByteArrayDecoder<T> {
- fn set_data(&mut self, data: ByteBufferPtr, num_values: usize) -> Result<()> {
- match T::get_physical_type() {
- Type::BYTE_ARRAY | Type::FIXED_LEN_BYTE_ARRAY => {
- let mut prefix_len_decoder = DeltaBitPackDecoder::<Int32Type>::new();
- prefix_len_decoder.set_data(data.all(), num_values)?;
- let num_prefixes = prefix_len_decoder.values_left();
- self.prefix_lengths.resize(num_prefixes, 0);
- prefix_len_decoder.get(&mut self.prefix_lengths[..])?;
-
- let mut suffix_decoder = DeltaLengthByteArrayDecoder::new();
- suffix_decoder
- .set_data(data.start_from(prefix_len_decoder.get_offset()), num_values)?;
- self.suffix_decoder = Some(suffix_decoder);
- self.num_values = num_prefixes;
- self.current_idx = 0;
- self.previous_value.clear();
- Ok(())
- }
- _ => {
- Err(general_err!(
- "DeltaByteArrayDecoder only supports ByteArrayType and FixedLenByteArrayType"
- ))
- }
- }
- }
-
- fn get(&mut self, buffer: &mut [T::T]) -> Result<usize> {
- match T::get_physical_type() {
- ty @ Type::BYTE_ARRAY | ty @ Type::FIXED_LEN_BYTE_ARRAY => {
- let num_values = cmp::min(buffer.len(), self.num_values);
- let mut v: [ByteArray; 1] = [ByteArray::new(); 1];
- for i in 0..num_values {
- // Process suffix
- // TODO: this is awkward - maybe we should add a non-vectorized API?
- let suffix_decoder = self.suffix_decoder.as_mut().expect("decoder not initialized");
- suffix_decoder.get(&mut v[..])?;
- let suffix = v[0].data();
-
- // Extract current prefix length, can be 0
- let prefix_len = self.prefix_lengths[self.current_idx] as usize;
-
- // Concatenate prefix with suffix
- let mut result = Vec::new();
- result.extend_from_slice(&self.previous_value[0..prefix_len]);
- result.extend_from_slice(suffix);
-
- let data = ByteBufferPtr::new(result.clone());
-
- match ty {
- Type::BYTE_ARRAY => buffer[i]
- .as_mut_any()
- .downcast_mut::<ByteArray>()
- .unwrap()
- .set_data(data),
- Type::FIXED_LEN_BYTE_ARRAY => buffer[i]
- .as_mut_any()
- .downcast_mut::<FixedLenByteArray>()
- .unwrap()
- .set_data(data),
- _ => unreachable!(),
- };
-
- self.previous_value = result;
- self.current_idx += 1;
- }
-
- self.num_values -= num_values;
- Ok(num_values)
- }
- _ => {
- Err(general_err!(
- "DeltaByteArrayDecoder only supports ByteArrayType and FixedLenByteArrayType"
- ))
- }
- }
- }
-
- fn values_left(&self) -> usize {
- self.num_values
- }
-
- fn encoding(&self) -> Encoding {
- Encoding::DELTA_BYTE_ARRAY
- }
-}
-
-#[cfg(test)]
-#[allow(clippy::approx_constant)]
-mod tests {
- use super::{super::encoding::*, *};
-
- use std::sync::Arc;
-
- use crate::schema::types::{
- ColumnDescPtr, ColumnDescriptor, ColumnPath, Type as SchemaType,
- };
- use crate::util::{
- bit_util::set_array_bit, memory::MemTracker, test_common::RandGen,
- };
-
- #[test]
- fn test_get_decoders() {
- // supported encodings
- create_and_check_decoder::<Int32Type>(Encoding::PLAIN, None);
- create_and_check_decoder::<Int32Type>(Encoding::DELTA_BINARY_PACKED, None);
- create_and_check_decoder::<Int32Type>(Encoding::DELTA_LENGTH_BYTE_ARRAY, None);
- create_and_check_decoder::<Int32Type>(Encoding::DELTA_BYTE_ARRAY, None);
- create_and_check_decoder::<BoolType>(Encoding::RLE, None);
-
- // error when initializing
- create_and_check_decoder::<Int32Type>(
- Encoding::RLE_DICTIONARY,
- Some(general_err!(
- "Cannot initialize this encoding through this function"
- )),
- );
- create_and_check_decoder::<Int32Type>(
- Encoding::PLAIN_DICTIONARY,
- Some(general_err!(
- "Cannot initialize this encoding through this function"
- )),
- );
-
- // unsupported
- create_and_check_decoder::<Int32Type>(
- Encoding::BIT_PACKED,
- Some(nyi_err!("Encoding BIT_PACKED is not supported")),
- );
- }
-
- #[test]
- fn test_plain_decode_int32() {
- let data = vec![42, 18, 52];
- let data_bytes = Int32Type::to_byte_array(&data[..]);
- let mut buffer = vec![0; 3];
- test_plain_decode::<Int32Type>(
- ByteBufferPtr::new(data_bytes),
- 3,
- -1,
- &mut buffer[..],
- &data[..],
- );
- }
-
- #[test]
- fn test_plain_decode_int32_spaced() {
- let data = [42, 18, 52];
- let expected_data = [0, 42, 0, 18, 0, 0, 52, 0];
- let data_bytes = Int32Type::to_byte_array(&data[..]);
- let mut buffer = vec![0; 8];
- let num_nulls = 5;
- let valid_bits = [0b01001010];
- test_plain_decode_spaced::<Int32Type>(
- ByteBufferPtr::new(data_bytes),
- 3,
- -1,
- &mut buffer[..],
- num_nulls,
- &valid_bits,
- &expected_data[..],
- );
- }
-
- #[test]
- fn test_plain_decode_int64() {
- let data = vec![42, 18, 52];
- let data_bytes = Int64Type::to_byte_array(&data[..]);
- let mut buffer = vec![0; 3];
- test_plain_decode::<Int64Type>(
- ByteBufferPtr::new(data_bytes),
- 3,
- -1,
- &mut buffer[..],
- &data[..],
- );
- }
-
- #[test]
- fn test_plain_decode_float() {
- let data = vec![3.14, 2.414, 12.51];
- let data_bytes = FloatType::to_byte_array(&data[..]);
- let mut buffer = vec![0.0; 3];
- test_plain_decode::<FloatType>(
- ByteBufferPtr::new(data_bytes),
- 3,
- -1,
- &mut buffer[..],
- &data[..],
- );
- }
-
- #[test]
- fn test_plain_decode_double() {
- let data = vec![3.14f64, 2.414f64, 12.51f64];
- let data_bytes = DoubleType::to_byte_array(&data[..]);
- let mut buffer = vec![0.0f64; 3];
- test_plain_decode::<DoubleType>(
- ByteBufferPtr::new(data_bytes),
- 3,
- -1,
- &mut buffer[..],
- &data[..],
- );
- }
-
- #[test]
- fn test_plain_decode_int96() {
- let mut data = vec![Int96::new(); 4];
- data[0].set_data(11, 22, 33);
- data[1].set_data(44, 55, 66);
- data[2].set_data(10, 20, 30);
- data[3].set_data(40, 50, 60);
- let data_bytes = Int96Type::to_byte_array(&data[..]);
- let mut buffer = vec![Int96::new(); 4];
- test_plain_decode::<Int96Type>(
- ByteBufferPtr::new(data_bytes),
- 4,
- -1,
- &mut buffer[..],
- &data[..],
- );
- }
-
- #[test]
- fn test_plain_decode_bool() {
- let data = vec![
- false, true, false, false, true, false, true, true, false, true,
- ];
- let data_bytes = BoolType::to_byte_array(&data[..]);
- let mut buffer = vec![false; 10];
- test_plain_decode::<BoolType>(
- ByteBufferPtr::new(data_bytes),
- 10,
- -1,
- &mut buffer[..],
- &data[..],
- );
- }
-
- #[test]
- fn test_plain_decode_byte_array() {
- let mut data = vec![ByteArray::new(); 2];
- data[0].set_data(ByteBufferPtr::new(String::from("hello").into_bytes()));
- data[1].set_data(ByteBufferPtr::new(String::from("parquet").into_bytes()));
- let data_bytes = ByteArrayType::to_byte_array(&data[..]);
- let mut buffer = vec![ByteArray::new(); 2];
- test_plain_decode::<ByteArrayType>(
- ByteBufferPtr::new(data_bytes),
- 2,
- -1,
- &mut buffer[..],
- &data[..],
- );
- }
-
- #[test]
- fn test_plain_decode_fixed_len_byte_array() {
- let mut data = vec![FixedLenByteArray::default(); 3];
- data[0].set_data(ByteBufferPtr::new(String::from("bird").into_bytes()));
- data[1].set_data(ByteBufferPtr::new(String::from("come").into_bytes()));
- data[2].set_data(ByteBufferPtr::new(String::from("flow").into_bytes()));
- let data_bytes = FixedLenByteArrayType::to_byte_array(&data[..]);
- let mut buffer = vec![FixedLenByteArray::default(); 3];
- test_plain_decode::<FixedLenByteArrayType>(
- ByteBufferPtr::new(data_bytes),
- 3,
- 4,
- &mut buffer[..],
- &data[..],
- );
- }
-
- fn test_plain_decode<T: DataType>(
- data: ByteBufferPtr,
- num_values: usize,
- type_length: i32,
- buffer: &mut [T::T],
- expected: &[T::T],
- ) {
- let mut decoder: PlainDecoder<T> = PlainDecoder::new(type_length);
- let result = decoder.set_data(data, num_values);
- assert!(result.is_ok());
- let result = decoder.get(buffer);
- assert!(result.is_ok());
- assert_eq!(decoder.values_left(), 0);
- assert_eq!(buffer, expected);
- }
-
- fn test_plain_decode_spaced<T: DataType>(
- data: ByteBufferPtr,
- num_values: usize,
- type_length: i32,
- buffer: &mut [T::T],
- num_nulls: usize,
- valid_bits: &[u8],
- expected: &[T::T],
- ) {
- let mut decoder: PlainDecoder<T> = PlainDecoder::new(type_length);
- let result = decoder.set_data(data, num_values);
- assert!(result.is_ok());
- let result = decoder.get_spaced(buffer, num_nulls, valid_bits);
- assert!(result.is_ok());
- assert_eq!(num_values + num_nulls, result.unwrap());
- assert_eq!(decoder.values_left(), 0);
- assert_eq!(buffer, expected);
- }
-
- #[test]
- #[should_panic(expected = "RleValueEncoder only supports BoolType")]
- fn test_rle_value_encode_int32_not_supported() {
- let mut encoder = RleValueEncoder::<Int32Type>::new();
- encoder.put(&[1, 2, 3, 4]).unwrap();
- }
-
- #[test]
- #[should_panic(expected = "RleValueDecoder only supports BoolType")]
- fn test_rle_value_decode_int32_not_supported() {
- let mut decoder = RleValueDecoder::<Int32Type>::new();
- decoder
- .set_data(ByteBufferPtr::new(vec![5, 0, 0, 0]), 1)
- .unwrap();
- }
-
- #[test]
- fn test_rle_value_decode_bool_decode() {
- // Test multiple 'put' calls on the same encoder
- let data = vec![
- BoolType::gen_vec(-1, 256),
- BoolType::gen_vec(-1, 257),
- BoolType::gen_vec(-1, 126),
- ];
- test_rle_value_decode::<BoolType>(data);
- }
-
- #[test]
- #[should_panic(expected = "Bit reader is not initialized")]
- fn test_delta_bit_packed_not_initialized_offset() {
- // Fail if set_data() is not called before get_offset()
- let decoder = DeltaBitPackDecoder::<Int32Type>::new();
- decoder.get_offset();
- }
-
- #[test]
- #[should_panic(expected = "Bit reader is not initialized")]
- fn test_delta_bit_packed_not_initialized_get() {
- // Fail if set_data() is not called before get()
- let mut decoder = DeltaBitPackDecoder::<Int32Type>::new();
- let mut buffer = vec![];
- decoder.get(&mut buffer).unwrap();
- }
-
- #[test]
- fn test_delta_bit_packed_int32_empty() {
- let data = vec![vec![0; 0]];
- test_delta_bit_packed_decode::<Int32Type>(data);
- }
-
- #[test]
- fn test_delta_bit_packed_int32_repeat() {
- let block_data = vec![
- 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8, 1, 2,
- 3, 4, 5, 6, 7, 8,
- ];
- test_delta_bit_packed_decode::<Int32Type>(vec![block_data]);
- }
-
- #[test]
- fn test_delta_bit_packed_int32_uneven() {
- let block_data = vec![1, -2, 3, -4, 5, 6, 7, 8, 9, 10, 11];
- test_delta_bit_packed_decode::<Int32Type>(vec![block_data]);
- }
-
- #[test]
- fn test_delta_bit_packed_int32_same_values() {
- let block_data = vec![
- 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127,
- 127,
- ];
- test_delta_bit_packed_decode::<Int32Type>(vec![block_data]);
-
- let block_data = vec![
- -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127, -127,
- -127, -127, -127,
- ];
- test_delta_bit_packed_decode::<Int32Type>(vec![block_data]);
- }
-
- #[test]
- fn test_delta_bit_packed_int32_min_max() {
- let block_data = vec![
- i32::min_value(),
- i32::max_value(),
- i32::min_value(),
- i32::max_value(),
- i32::min_value(),
- i32::max_value(),
- i32::min_value(),
- i32::max_value(),
- ];
- test_delta_bit_packed_decode::<Int32Type>(vec![block_data]);
- }
-
- #[test]
- fn test_delta_bit_packed_int32_multiple_blocks() {
- // Test multiple 'put' calls on the same encoder
- let data = vec![
- Int32Type::gen_vec(-1, 64),
- Int32Type::gen_vec(-1, 128),
- Int32Type::gen_vec(-1, 64),
- ];
- test_delta_bit_packed_decode::<Int32Type>(data);
- }
-
- #[test]
- fn test_delta_bit_packed_int32_data_across_blocks() {
- // Test multiple 'put' calls on the same encoder
- let data = vec![Int32Type::gen_vec(-1, 256), Int32Type::gen_vec(-1, 257)];
- test_delta_bit_packed_decode::<Int32Type>(data);
- }
-
- #[test]
- fn test_delta_bit_packed_int32_with_empty_blocks() {
- let data = vec![
- Int32Type::gen_vec(-1, 128),
- vec![0; 0],
- Int32Type::gen_vec(-1, 64),
- ];
- test_delta_bit_packed_decode::<Int32Type>(data);
- }
-
- #[test]
- fn test_delta_bit_packed_int64_empty() {
- let data = vec![vec![0; 0]];
- test_delta_bit_packed_decode::<Int64Type>(data);
- }
-
- #[test]
- fn test_delta_bit_packed_int64_min_max() {
- let block_data = vec![
- i64::min_value(),
- i64::max_value(),
- i64::min_value(),
- i64::max_value(),
- i64::min_value(),
- i64::max_value(),
- i64::min_value(),
- i64::max_value(),
- ];
- test_delta_bit_packed_decode::<Int64Type>(vec![block_data]);
- }
-
- #[test]
- fn test_delta_bit_packed_int64_multiple_blocks() {
- // Test multiple 'put' calls on the same encoder
- let data = vec![
- Int64Type::gen_vec(-1, 64),
- Int64Type::gen_vec(-1, 128),
- Int64Type::gen_vec(-1, 64),
- ];
- test_delta_bit_packed_decode::<Int64Type>(data);
- }
-
- #[test]
- fn test_delta_bit_packed_decoder_sample() {
- let data_bytes = vec![
- 128, 1, 4, 3, 58, 28, 6, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- ];
- let buffer = ByteBufferPtr::new(data_bytes);
- let mut decoder: DeltaBitPackDecoder<Int32Type> = DeltaBitPackDecoder::new();
- decoder.set_data(buffer, 3).unwrap();
- // check exact offsets, because when reading partial values we end up with
- // some data not being read from bit reader
- assert_eq!(decoder.get_offset(), 5);
- let mut result = vec![0, 0, 0];
- decoder.get(&mut result).unwrap();
- assert_eq!(decoder.get_offset(), 34);
- assert_eq!(result, vec![29, 43, 89]);
- }
-
- #[test]
- fn test_delta_byte_array_same_arrays() {
- let data = vec![
- vec![ByteArray::from(vec![1, 2, 3, 4, 5, 6])],
- vec![
- ByteArray::from(vec![1, 2, 3, 4, 5, 6]),
- ByteArray::from(vec![1, 2, 3, 4, 5, 6]),
- ],
- vec![
- ByteArray::from(vec![1, 2, 3, 4, 5, 6]),
- ByteArray::from(vec![1, 2, 3, 4, 5, 6]),
- ],
- ];
- test_delta_byte_array_decode(data);
- }
-
- #[test]
- fn test_delta_byte_array_unique_arrays() {
- let data = vec![
- vec![ByteArray::from(vec![1])],
- vec![ByteArray::from(vec![2, 3]), ByteArray::from(vec![4, 5, 6])],
- vec![
- ByteArray::from(vec![7, 8]),
- ByteArray::from(vec![9, 0, 1, 2]),
- ],
- ];
- test_delta_byte_array_decode(data);
- }
-
- #[test]
- fn test_delta_byte_array_single_array() {
- let data = vec![vec![ByteArray::from(vec![1, 2, 3, 4, 5, 6])]];
- test_delta_byte_array_decode(data);
- }
-
- fn test_rle_value_decode<T: DataType>(data: Vec<Vec<T::T>>) {
- test_encode_decode::<T>(data, Encoding::RLE);
- }
-
- fn test_delta_bit_packed_decode<T: DataType>(data: Vec<Vec<T::T>>) {
- test_encode_decode::<T>(data, Encoding::DELTA_BINARY_PACKED);
- }
-
- fn test_delta_byte_array_decode(data: Vec<Vec<ByteArray>>) {
- test_encode_decode::<ByteArrayType>(data, Encoding::DELTA_BYTE_ARRAY);
- }
-
- // Input data represents vector of data slices to write (test multiple `put()` calls)
- // For example,
- // vec![vec![1, 2, 3]] invokes `put()` once and writes {1, 2, 3}
- // vec![vec![1, 2], vec![3]] invokes `put()` twice and writes {1, 2, 3}
- fn test_encode_decode<T: DataType>(data: Vec<Vec<T::T>>, encoding: Encoding) {
- // Type length should not really matter for encode/decode test,
- // otherwise change it based on type
- let col_descr = create_test_col_desc_ptr(-1, T::get_physical_type());
-
- // Encode data
- let mut encoder =
- get_encoder::<T>(col_descr.clone(), encoding, Arc::new(MemTracker::new()))
- .expect("get encoder");
-
- for v in &data[..] {
- encoder.put(&v[..]).expect("ok to encode");
- }
- let bytes = encoder.flush_buffer().expect("ok to flush buffer");
-
- // Flatten expected data as contiguous array of values
- let expected: Vec<T::T> = data.iter().flat_map(|s| s.clone()).collect();
-
- // Decode data and compare with original
- let mut decoder = get_decoder::<T>(col_descr, encoding).expect("get decoder");
-
- let mut result = vec![T::T::default(); expected.len()];
- decoder
- .set_data(bytes, expected.len())
- .expect("ok to set data");
- let mut result_num_values = 0;
- while decoder.values_left() > 0 {
- result_num_values += decoder
- .get(&mut result[result_num_values..])
- .expect("ok to decode");
- }
- assert_eq!(result_num_values, expected.len());
- assert_eq!(result, expected);
- }
-
- fn create_and_check_decoder<T: DataType>(
- encoding: Encoding,
- err: Option<ParquetError>,
- ) {
- let descr = create_test_col_desc_ptr(-1, T::get_physical_type());
- let decoder = get_decoder::<T>(descr, encoding);
- match err {
- Some(parquet_error) => {
- assert!(decoder.is_err());
- assert_eq!(decoder.err().unwrap(), parquet_error);
- }
- None => {
- assert!(decoder.is_ok());
- assert_eq!(decoder.unwrap().encoding(), encoding);
- }
- }
- }
-
- // Creates test column descriptor.
- fn create_test_col_desc_ptr(type_len: i32, t: Type) -> ColumnDescPtr {
- let ty = SchemaType::primitive_type_builder("t", t)
- .with_length(type_len)
- .build()
- .unwrap();
- Arc::new(ColumnDescriptor::new(
- Arc::new(ty),
- 0,
- 0,
- ColumnPath::new(vec![]),
- ))
- }
-
- fn usize_to_bytes(v: usize) -> [u8; 4] {
- (v as u32).to_ne_bytes()
- }
-
- /// A util trait to convert slices of different types to byte arrays
- trait ToByteArray<T: DataType> {
- #[allow(clippy::wrong_self_convention)]
- fn to_byte_array(data: &[T::T]) -> Vec<u8>;
- }
-
- macro_rules! to_byte_array_impl {
- ($ty: ty) => {
- impl ToByteArray<$ty> for $ty {
- fn to_byte_array(data: &[<$ty as DataType>::T]) -> Vec<u8> {
- <$ty as DataType>::T::slice_as_bytes(data).to_vec()
- }
- }
- };
- }
-
- to_byte_array_impl!(Int32Type);
- to_byte_array_impl!(Int64Type);
- to_byte_array_impl!(FloatType);
- to_byte_array_impl!(DoubleType);
-
- impl ToByteArray<BoolType> for BoolType {
- fn to_byte_array(data: &[bool]) -> Vec<u8> {
- let mut v = vec![];
- for i in 0..data.len() {
- if i % 8 == 0 {
- v.push(0);
- }
- if data[i] {
- set_array_bit(&mut v[..], i);
- }
- }
- v
- }
- }
-
- impl ToByteArray<Int96Type> for Int96Type {
- fn to_byte_array(data: &[Int96]) -> Vec<u8> {
- let mut v = vec![];
- for d in data {
- v.extend_from_slice(d.as_bytes());
- }
- v
- }
- }
-
- impl ToByteArray<ByteArrayType> for ByteArrayType {
- fn to_byte_array(data: &[ByteArray]) -> Vec<u8> {
- let mut v = vec![];
- for d in data {
- let buf = d.data();
- let len = &usize_to_bytes(buf.len());
- v.extend_from_slice(len);
- v.extend(buf);
- }
- v
- }
- }
-
- impl ToByteArray<FixedLenByteArrayType> for FixedLenByteArrayType {
- fn to_byte_array(data: &[FixedLenByteArray]) -> Vec<u8> {
- let mut v = vec![];
- for d in data {
- let buf = d.data();
- v.extend(buf);
- }
- v
- }
- }
-}
diff --git a/parquet/src/encodings/encoding.rs b/parquet/src/encodings/encoding.rs
deleted file mode 100644
index d042738..0000000
--- a/parquet/src/encodings/encoding.rs
+++ /dev/null
@@ -1,1334 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-//! Contains all supported encoders for Parquet.
-
-use std::{cmp, io::Write, marker::PhantomData};
-
-use crate::basic::*;
-use crate::data_type::private::ParquetValueType;
-use crate::data_type::*;
-use crate::encodings::rle::RleEncoder;
-use crate::errors::{ParquetError, Result};
-use crate::schema::types::ColumnDescPtr;
-use crate::util::{
- bit_util::{self, log2, num_required_bits, BitWriter},
- hash_util,
- memory::{Buffer, ByteBuffer, ByteBufferPtr, MemTrackerPtr},
-};
-
-// ----------------------------------------------------------------------
-// Encoders
-
-/// An Parquet encoder for the data type `T`.
-///
-/// Currently this allocates internal buffers for the encoded values. After done putting
-/// values, caller should call `flush_buffer()` to get an immutable buffer pointer.
-pub trait Encoder<T: DataType> {
- /// Encodes data from `values`.
- fn put(&mut self, values: &[T::T]) -> Result<()>;
-
- /// Encodes data from `values`, which contains spaces for null values, that is
- /// identified by `valid_bits`.
- ///
- /// Returns the number of non-null values encoded.
- fn put_spaced(&mut self, values: &[T::T], valid_bits: &[u8]) -> Result<usize> {
- let num_values = values.len();
- let mut buffer = Vec::with_capacity(num_values);
- // TODO: this is pretty inefficient. Revisit in future.
- for i in 0..num_values {
- if bit_util::get_bit(valid_bits, i) {
- buffer.push(values[i].clone());
- }
- }
- self.put(&buffer[..])?;
- Ok(buffer.len())
- }
-
- /// Returns the encoding type of this encoder.
- fn encoding(&self) -> Encoding;
-
- /// Returns an estimate of the encoded data, in bytes.
- /// Method call must be O(1).
- fn estimated_data_encoded_size(&self) -> usize;
-
- /// Flushes the underlying byte buffer that's being processed by this encoder, and
- /// return the immutable copy of it. This will also reset the internal state.
- fn flush_buffer(&mut self) -> Result<ByteBufferPtr>;
-}
-
-/// Gets a encoder for the particular data type `T` and encoding `encoding`. Memory usage
-/// for the encoder instance is tracked by `mem_tracker`.
-pub fn get_encoder<T: DataType>(
- desc: ColumnDescPtr,
- encoding: Encoding,
- mem_tracker: MemTrackerPtr,
-) -> Result<Box<dyn Encoder<T>>> {
- let encoder: Box<dyn Encoder<T>> = match encoding {
- Encoding::PLAIN => Box::new(PlainEncoder::new(desc, mem_tracker, vec![])),
- Encoding::RLE_DICTIONARY | Encoding::PLAIN_DICTIONARY => {
- return Err(general_err!(
- "Cannot initialize this encoding through this function"
- ));
- }
- Encoding::RLE => Box::new(RleValueEncoder::new()),
- Encoding::DELTA_BINARY_PACKED => Box::new(DeltaBitPackEncoder::new()),
- Encoding::DELTA_LENGTH_BYTE_ARRAY => Box::new(DeltaLengthByteArrayEncoder::new()),
- Encoding::DELTA_BYTE_ARRAY => Box::new(DeltaByteArrayEncoder::new()),
- e => return Err(nyi_err!("Encoding {} is not supported", e)),
- };
- Ok(encoder)
-}
-
-// ----------------------------------------------------------------------
-// Plain encoding
-
-/// Plain encoding that supports all types.
-/// Values are encoded back to back.
-/// The plain encoding is used whenever a more efficient encoding can not be used.
-/// It stores the data in the following format:
-/// - BOOLEAN - 1 bit per value, 0 is false; 1 is true.
-/// - INT32 - 4 bytes per value, stored as little-endian.
-/// - INT64 - 8 bytes per value, stored as little-endian.
-/// - FLOAT - 4 bytes per value, stored as IEEE little-endian.
-/// - DOUBLE - 8 bytes per value, stored as IEEE little-endian.
-/// - BYTE_ARRAY - 4 byte length stored as little endian, followed by bytes.
-/// - FIXED_LEN_BYTE_ARRAY - just the bytes are stored.
-pub struct PlainEncoder<T: DataType> {
- buffer: ByteBuffer,
- bit_writer: BitWriter,
- desc: ColumnDescPtr,
- _phantom: PhantomData<T>,
-}
-
-impl<T: DataType> PlainEncoder<T> {
- /// Creates new plain encoder.
- pub fn new(desc: ColumnDescPtr, mem_tracker: MemTrackerPtr, vec: Vec<u8>) -> Self {
- let mut byte_buffer = ByteBuffer::new().with_mem_tracker(mem_tracker);
- byte_buffer.set_data(vec);
- Self {
- buffer: byte_buffer,
- bit_writer: BitWriter::new(256),
- desc,
- _phantom: PhantomData,
- }
- }
-}
-
-impl<T: DataType> Encoder<T> for PlainEncoder<T> {
- // Performance Note:
- // As far as can be seen these functions are rarely called and as such we can hint to the
- // compiler that they dont need to be folded into hot locations in the final output.
- #[cold]
- fn encoding(&self) -> Encoding {
- Encoding::PLAIN
- }
-
- fn estimated_data_encoded_size(&self) -> usize {
- self.buffer.size() + self.bit_writer.bytes_written()
- }
-
- #[inline]
- fn flush_buffer(&mut self) -> Result<ByteBufferPtr> {
- self.buffer.write_all(self.bit_writer.flush_buffer())?;
- self.buffer.flush()?;
- self.bit_writer.clear();
-
- Ok(self.buffer.consume())
- }
-
- #[inline]
- fn put(&mut self, values: &[T::T]) -> Result<()> {
- T::T::encode(values, &mut self.buffer, &mut self.bit_writer)?;
- Ok(())
- }
-}
-
-// ----------------------------------------------------------------------
-// Dictionary encoding
-
-const INITIAL_HASH_TABLE_SIZE: usize = 1024;
-const MAX_HASH_LOAD: f32 = 0.7;
-const HASH_SLOT_EMPTY: i32 = -1;
-
-/// Dictionary encoder.
-/// The dictionary encoding builds a dictionary of values encountered in a given column.
-/// The dictionary page is written first, before the data pages of the column chunk.
-///
-/// Dictionary page format: the entries in the dictionary - in dictionary order -
-/// using the plain encoding.
-///
-/// Data page format: the bit width used to encode the entry ids stored as 1 byte
-/// (max bit width = 32), followed by the values encoded using RLE/Bit packed described
-/// above (with the given bit width).
-pub struct DictEncoder<T: DataType> {
- // Descriptor for the column to be encoded.
- desc: ColumnDescPtr,
-
- // Size of the table. **Must be** a power of 2.
- hash_table_size: usize,
-
- // Store `hash_table_size` - 1, so that `j & mod_bitmask` is equivalent to
- // `j % hash_table_size`, but uses far fewer CPU cycles.
- mod_bitmask: u32,
-
- // Stores indices which map (many-to-one) to the values in the `uniques` array.
- // Here we are using fix-sized array with linear probing.
- // A slot with `HASH_SLOT_EMPTY` indicates the slot is not currently occupied.
- hash_slots: Buffer<i32>,
-
- // Indices that have not yet be written out by `write_indices()`.
- buffered_indices: Buffer<i32>,
-
- // The unique observed values.
- uniques: Buffer<T::T>,
-
- // Size in bytes needed to encode this dictionary.
- uniques_size_in_bytes: usize,
-
- // Tracking memory usage for the various data structures in this struct.
- mem_tracker: MemTrackerPtr,
-}
-
-impl<T: DataType> DictEncoder<T> {
- /// Creates new dictionary encoder.
- pub fn new(desc: ColumnDescPtr, mem_tracker: MemTrackerPtr) -> Self {
- let mut slots = Buffer::new().with_mem_tracker(mem_tracker.clone());
- slots.resize(INITIAL_HASH_TABLE_SIZE, -1);
- Self {
- desc,
- hash_table_size: INITIAL_HASH_TABLE_SIZE,
- mod_bitmask: (INITIAL_HASH_TABLE_SIZE - 1) as u32,
- hash_slots: slots,
- buffered_indices: Buffer::new().with_mem_tracker(mem_tracker.clone()),
- uniques: Buffer::new().with_mem_tracker(mem_tracker.clone()),
- uniques_size_in_bytes: 0,
- mem_tracker,
- }
- }
-
- /// Returns true if dictionary entries are sorted, false otherwise.
- #[inline]
- pub fn is_sorted(&self) -> bool {
- // Sorting is not supported currently.
- false
- }
-
- /// Returns number of unique values (keys) in the dictionary.
- pub fn num_entries(&self) -> usize {
- self.uniques.size()
- }
-
- /// Returns size of unique values (keys) in the dictionary, in bytes.
- pub fn dict_encoded_size(&self) -> usize {
- self.uniques_size_in_bytes
- }
-
- /// Writes out the dictionary values with PLAIN encoding in a byte buffer, and return
- /// the result.
- #[inline]
- pub fn write_dict(&self) -> Result<ByteBufferPtr> {
- let mut plain_encoder =
- PlainEncoder::<T>::new(self.desc.clone(), self.mem_tracker.clone(), vec![]);
- plain_encoder.put(self.uniques.data())?;
- plain_encoder.flush_buffer()
- }
-
- /// Writes out the dictionary values with RLE encoding in a byte buffer, and return
- /// the result.
- pub fn write_indices(&mut self) -> Result<ByteBufferPtr> {
- // TODO: the caller should allocate the buffer
- let buffer_len = self.estimated_data_encoded_size();
- let mut buffer: Vec<u8> = vec![0; buffer_len as usize];
- buffer[0] = self.bit_width() as u8;
- self.mem_tracker.alloc(buffer.capacity() as i64);
-
- // Write bit width in the first byte
- buffer.write_all((self.bit_width() as u8).as_bytes())?;
- let mut encoder = RleEncoder::new_from_buf(self.bit_width(), buffer, 1);
- for index in self.buffered_indices.data() {
- if !encoder.put(*index as u64)? {
- return Err(general_err!("Encoder doesn't have enough space"));
- }
- }
- self.buffered_indices.clear();
- Ok(ByteBufferPtr::new(encoder.consume()?))
- }
-
- #[inline]
- #[allow(clippy::unnecessary_wraps)]
- fn put_one(&mut self, value: &T::T) -> Result<()> {
- let mut j = (hash_util::hash(value, 0) & self.mod_bitmask) as usize;
- let mut index = self.hash_slots[j];
-
- while index != HASH_SLOT_EMPTY && self.uniques[index as usize] != *value {
- j += 1;
- if j == self.hash_table_size {
- j = 0;
- }
- index = self.hash_slots[j];
- }
-
- if index == HASH_SLOT_EMPTY {
- index = self.insert_fresh_slot(j, value.clone());
- }
-
- self.buffered_indices.push(index);
- Ok(())
- }
-
- #[inline(never)]
- fn insert_fresh_slot(&mut self, slot: usize, value: T::T) -> i32 {
- let index = self.uniques.size() as i32;
- self.hash_slots[slot] = index;
-
- let (base_size, num_elements) = value.dict_encoding_size();
-
- let unique_size = match T::get_physical_type() {
- Type::BYTE_ARRAY => base_size + num_elements,
- Type::FIXED_LEN_BYTE_ARRAY => self.desc.type_length() as usize,
- _ => base_size,
- };
-
- self.uniques_size_in_bytes += unique_size;
- self.uniques.push(value);
-
- if self.uniques.size() > (self.hash_table_size as f32 * MAX_HASH_LOAD) as usize {
- self.double_table_size();
- }
-
- index
- }
-
- #[inline]
- fn bit_width(&self) -> u8 {
- let num_entries = self.uniques.size();
- if num_entries == 0 {
- 0
- } else if num_entries == 1 {
- 1
- } else {
- log2(num_entries as u64) as u8
- }
- }
-
- fn double_table_size(&mut self) {
- let new_size = self.hash_table_size * 2;
- let mut new_hash_slots = Buffer::new().with_mem_tracker(self.mem_tracker.clone());
- new_hash_slots.resize(new_size, HASH_SLOT_EMPTY);
- for i in 0..self.hash_table_size {
- let index = self.hash_slots[i];
- if index == HASH_SLOT_EMPTY {
- continue;
- }
- let value = &self.uniques[index as usize];
- let mut j = (hash_util::hash(value, 0) & ((new_size - 1) as u32)) as usize;
- let mut slot = new_hash_slots[j];
- while slot != HASH_SLOT_EMPTY && self.uniques[slot as usize] != *value {
- j += 1;
- if j == new_size {
- j = 0;
- }
- slot = new_hash_slots[j];
- }
-
- new_hash_slots[j] = index;
- }
-
- self.hash_table_size = new_size;
- self.mod_bitmask = (new_size - 1) as u32;
- self.hash_slots = new_hash_slots;
- }
-}
-
-impl<T: DataType> Encoder<T> for DictEncoder<T> {
- #[inline]
- fn put(&mut self, values: &[T::T]) -> Result<()> {
- for i in values {
- self.put_one(&i)?
- }
- Ok(())
- }
-
- // Performance Note:
- // As far as can be seen these functions are rarely called and as such we can hint to the
- // compiler that they dont need to be folded into hot locations in the final output.
- #[cold]
- fn encoding(&self) -> Encoding {
- Encoding::PLAIN_DICTIONARY
- }
-
- #[inline]
- fn estimated_data_encoded_size(&self) -> usize {
- let bit_width = self.bit_width();
- 1 + RleEncoder::min_buffer_size(bit_width)
- + RleEncoder::max_buffer_size(bit_width, self.buffered_indices.size())
- }
-
- #[inline]
- fn flush_buffer(&mut self) -> Result<ByteBufferPtr> {
- self.write_indices()
- }
-}
-
-// ----------------------------------------------------------------------
-// RLE encoding
-
-const DEFAULT_RLE_BUFFER_LEN: usize = 1024;
-
-/// RLE/Bit-Packing hybrid encoding for values.
-/// Currently is used only for data pages v2 and supports boolean types.
-pub struct RleValueEncoder<T: DataType> {
- // Buffer with raw values that we collect,
- // when flushing buffer they are encoded using RLE encoder
- encoder: Option<RleEncoder>,
- _phantom: PhantomData<T>,
-}
-
-impl<T: DataType> RleValueEncoder<T> {
- /// Creates new rle value encoder.
- pub fn new() -> Self {
- Self {
- encoder: None,
- _phantom: PhantomData,
- }
- }
-}
-
-impl<T: DataType> Encoder<T> for RleValueEncoder<T> {
- #[inline]
- fn put(&mut self, values: &[T::T]) -> Result<()> {
- ensure_phys_ty!(Type::BOOLEAN, "RleValueEncoder only supports BoolType");
-
- if self.encoder.is_none() {
- self.encoder = Some(RleEncoder::new(1, DEFAULT_RLE_BUFFER_LEN));
- }
- let rle_encoder = self.encoder.as_mut().unwrap();
- for value in values {
- let value = value.as_u64()?;
- if !rle_encoder.put(value)? {
- return Err(general_err!("RLE buffer is full"));
- }
- }
- Ok(())
- }
-
- // Performance Note:
- // As far as can be seen these functions are rarely called and as such we can hint to the
- // compiler that they dont need to be folded into hot locations in the final output.
- #[cold]
- fn encoding(&self) -> Encoding {
- Encoding::RLE
- }
-
- #[inline]
- fn estimated_data_encoded_size(&self) -> usize {
- match self.encoder {
- Some(ref enc) => enc.len(),
- None => 0,
- }
- }
-
- #[inline]
- fn flush_buffer(&mut self) -> Result<ByteBufferPtr> {
- ensure_phys_ty!(Type::BOOLEAN, "RleValueEncoder only supports BoolType");
- let rle_encoder = self
- .encoder
- .as_mut()
- .expect("RLE value encoder is not initialized");
-
- // Flush all encoder buffers and raw values
- let encoded_data = {
- let buf = rle_encoder.flush_buffer()?;
-
- // Note that buf does not have any offset, all data is encoded bytes
- let len = (buf.len() as i32).to_le();
- let len_bytes = len.as_bytes();
- let mut encoded_data = vec![];
- encoded_data.extend_from_slice(len_bytes);
- encoded_data.extend_from_slice(buf);
- encoded_data
- };
- // Reset rle encoder for the next batch
- rle_encoder.clear();
-
- Ok(ByteBufferPtr::new(encoded_data))
- }
-}
-
-// ----------------------------------------------------------------------
-// DELTA_BINARY_PACKED encoding
-
-const MAX_PAGE_HEADER_WRITER_SIZE: usize = 32;
-const MAX_BIT_WRITER_SIZE: usize = 10 * 1024 * 1024;
-const DEFAULT_BLOCK_SIZE: usize = 128;
-const DEFAULT_NUM_MINI_BLOCKS: usize = 4;
-
-/// Delta bit packed encoder.
-/// Consists of a header followed by blocks of delta encoded values binary packed.
-///
-/// Delta-binary-packing:
-/// ```shell
-/// [page-header] [block 1], [block 2], ... [block N]
-/// ```
-///
-/// Each page header consists of:
-/// ```shell
-/// [block size] [number of miniblocks in a block] [total value count] [first value]
-/// ```
-///
-/// Each block consists of:
-/// ```shell
-/// [min delta] [list of bitwidths of miniblocks] [miniblocks]
-/// ```
-///
-/// Current implementation writes values in `put` method, multiple calls to `put` to
-/// existing block or start new block if block size is exceeded. Calling `flush_buffer`
-/// writes out all data and resets internal state, including page header.
-///
-/// Supports only INT32 and INT64.
-pub struct DeltaBitPackEncoder<T: DataType> {
- page_header_writer: BitWriter,
- bit_writer: BitWriter,
- total_values: usize,
- first_value: i64,
- current_value: i64,
- block_size: usize,
- mini_block_size: usize,
- num_mini_blocks: usize,
- values_in_block: usize,
- deltas: Vec<i64>,
- _phantom: PhantomData<T>,
-}
-
-impl<T: DataType> DeltaBitPackEncoder<T> {
- /// Creates new delta bit packed encoder.
- pub fn new() -> Self {
- let block_size = DEFAULT_BLOCK_SIZE;
- let num_mini_blocks = DEFAULT_NUM_MINI_BLOCKS;
- let mini_block_size = block_size / num_mini_blocks;
- assert!(mini_block_size % 8 == 0);
- Self::assert_supported_type();
-
- DeltaBitPackEncoder {
- page_header_writer: BitWriter::new(MAX_PAGE_HEADER_WRITER_SIZE),
- bit_writer: BitWriter::new(MAX_BIT_WRITER_SIZE),
- total_values: 0,
- first_value: 0,
- current_value: 0, // current value to keep adding deltas
- block_size, // can write fewer values than block size for last block
- mini_block_size,
- num_mini_blocks,
- values_in_block: 0, // will be at most block_size
- deltas: vec![0; block_size],
- _phantom: PhantomData,
- }
- }
-
- /// Writes page header for blocks, this method is invoked when we are done encoding
- /// values. It is also okay to encode when no values have been provided
- fn write_page_header(&mut self) {
- // We ignore the result of each 'put' operation, because
- // MAX_PAGE_HEADER_WRITER_SIZE is chosen to fit all header values and
- // guarantees that writes will not fail.
-
- // Write the size of each block
- self.page_header_writer.put_vlq_int(self.block_size as u64);
- // Write the number of mini blocks
- self.page_header_writer
- .put_vlq_int(self.num_mini_blocks as u64);
- // Write the number of all values (including non-encoded first value)
- self.page_header_writer
- .put_vlq_int(self.total_values as u64);
- // Write first value
- self.page_header_writer.put_zigzag_vlq_int(self.first_value);
- }
-
- // Write current delta buffer (<= 'block size' values) into bit writer
- #[inline(never)]
- fn flush_block_values(&mut self) -> Result<()> {
- if self.values_in_block == 0 {
- return Ok(());
- }
-
- let mut min_delta = i64::max_value();
- for i in 0..self.values_in_block {
- min_delta = cmp::min(min_delta, self.deltas[i]);
- }
-
- // Write min delta
- self.bit_writer.put_zigzag_vlq_int(min_delta);
-
- // Slice to store bit width for each mini block
- let offset = self.bit_writer.skip(self.num_mini_blocks)?;
-
- for i in 0..self.num_mini_blocks {
- // Find how many values we need to encode - either block size or whatever
- // values left
- let n = cmp::min(self.mini_block_size, self.values_in_block);
- if n == 0 {
- break;
- }
-
- // Compute the max delta in current mini block
- let mut max_delta = i64::min_value();
- for j in 0..n {
- max_delta =
- cmp::max(max_delta, self.deltas[i * self.mini_block_size + j]);
- }
-
- // Compute bit width to store (max_delta - min_delta)
- let bit_width = num_required_bits(self.subtract_u64(max_delta, min_delta));
- self.bit_writer.write_at(offset + i, bit_width as u8);
-
- // Encode values in current mini block using min_delta and bit_width
- for j in 0..n {
- let packed_value = self
- .subtract_u64(self.deltas[i * self.mini_block_size + j], min_delta);
- self.bit_writer.put_value(packed_value, bit_width);
- }
-
- // Pad the last block (n < mini_block_size)
- for _ in n..self.mini_block_size {
- self.bit_writer.put_value(0, bit_width);
- }
-
- self.values_in_block -= n;
- }
-
- assert!(
- self.values_in_block == 0,
- "Expected 0 values in block, found {}",
- self.values_in_block
- );
- Ok(())
- }
-}
-
-// Implementation is shared between Int32Type and Int64Type,
-// see `DeltaBitPackEncoderConversion` below for specifics.
-impl<T: DataType> Encoder<T> for DeltaBitPackEncoder<T> {
- fn put(&mut self, values: &[T::T]) -> Result<()> {
- if values.is_empty() {
- return Ok(());
- }
-
- // Define values to encode, initialize state
- let mut idx = if self.total_values == 0 {
- self.first_value = self.as_i64(values, 0);
- self.current_value = self.first_value;
- 1
- } else {
- 0
- };
- // Add all values (including first value)
- self.total_values += values.len();
-
- // Write block
- while idx < values.len() {
- let value = self.as_i64(values, idx);
- self.deltas[self.values_in_block] = self.subtract(value, self.current_value);
- self.current_value = value;
- idx += 1;
- self.values_in_block += 1;
- if self.values_in_block == self.block_size {
- self.flush_block_values()?;
- }
- }
- Ok(())
- }
-
- // Performance Note:
- // As far as can be seen these functions are rarely called and as such we can hint to the
- // compiler that they dont need to be folded into hot locations in the final output.
- #[cold]
- fn encoding(&self) -> Encoding {
- Encoding::DELTA_BINARY_PACKED
- }
-
- fn estimated_data_encoded_size(&self) -> usize {
- self.bit_writer.bytes_written()
- }
-
- fn flush_buffer(&mut self) -> Result<ByteBufferPtr> {
- // Write remaining values
- self.flush_block_values()?;
- // Write page header with total values
- self.write_page_header();
-
- let mut buffer = ByteBuffer::new();
- buffer.write_all(self.page_header_writer.flush_buffer())?;
- buffer.write_all(self.bit_writer.flush_buffer())?;
- buffer.flush()?;
-
- // Reset state
- self.page_header_writer.clear();
- self.bit_writer.clear();
- self.total_values = 0;
- self.first_value = 0;
- self.current_value = 0;
- self.values_in_block = 0;
-
- Ok(buffer.consume())
- }
-}
-
-/// Helper trait to define specific conversions and subtractions when computing deltas
-trait DeltaBitPackEncoderConversion<T: DataType> {
- // Method should panic if type is not supported, otherwise no-op
- fn assert_supported_type();
-
- fn as_i64(&self, values: &[T::T], index: usize) -> i64;
-
- fn subtract(&self, left: i64, right: i64) -> i64;
-
- fn subtract_u64(&self, left: i64, right: i64) -> u64;
-}
-
-impl<T: DataType> DeltaBitPackEncoderConversion<T> for DeltaBitPackEncoder<T> {
- #[inline]
- fn assert_supported_type() {
- ensure_phys_ty!(
- Type::INT32 | Type::INT64,
- "DeltaBitPackDecoder only supports Int32Type and Int64Type"
- );
- }
-
- #[inline]
- fn as_i64(&self, values: &[T::T], index: usize) -> i64 {
- values[index]
- .as_i64()
- .expect("DeltaBitPackDecoder only supports Int32Type and Int64Type")
- }
-
- #[inline]
- fn subtract(&self, left: i64, right: i64) -> i64 {
- // It is okay for values to overflow, wrapping_sub wrapping around at the boundary
- match T::get_physical_type() {
- Type::INT32 => (left as i32).wrapping_sub(right as i32) as i64,
- Type::INT64 => left.wrapping_sub(right),
- _ => panic!("DeltaBitPackDecoder only supports Int32Type and Int64Type"),
- }
- }
-
- #[inline]
- fn subtract_u64(&self, left: i64, right: i64) -> u64 {
- match T::get_physical_type() {
- // Conversion of i32 -> u32 -> u64 is to avoid non-zero left most bytes in int repr
- Type::INT32 => (left as i32).wrapping_sub(right as i32) as u32 as u64,
- Type::INT64 => left.wrapping_sub(right) as u64,
- _ => panic!("DeltaBitPackDecoder only supports Int32Type and Int64Type"),
- }
- }
-}
-
-// ----------------------------------------------------------------------
-// DELTA_LENGTH_BYTE_ARRAY encoding
-
-/// Encoding for byte arrays to separate the length values and the data.
-/// The lengths are encoded using DELTA_BINARY_PACKED encoding, data is
-/// stored as raw bytes.
-pub struct DeltaLengthByteArrayEncoder<T: DataType> {
- // length encoder
- len_encoder: DeltaBitPackEncoder<Int32Type>,
- // byte array data
- data: Vec<ByteArray>,
- // data size in bytes of encoded values
- encoded_size: usize,
- _phantom: PhantomData<T>,
-}
-
-impl<T: DataType> DeltaLengthByteArrayEncoder<T> {
- /// Creates new delta length byte array encoder.
- pub fn new() -> Self {
- Self {
- len_encoder: DeltaBitPackEncoder::new(),
- data: vec![],
- encoded_size: 0,
- _phantom: PhantomData,
- }
- }
-}
-
-impl<T: DataType> Encoder<T> for DeltaLengthByteArrayEncoder<T> {
- fn put(&mut self, values: &[T::T]) -> Result<()> {
- ensure_phys_ty!(
- Type::BYTE_ARRAY | Type::FIXED_LEN_BYTE_ARRAY,
- "DeltaLengthByteArrayEncoder only supports ByteArrayType"
- );
-
- let val_it = || {
- values
- .iter()
- .map(|x| x.as_any().downcast_ref::<ByteArray>().unwrap())
- };
-
- let lengths: Vec<i32> =
- val_it().map(|byte_array| byte_array.len() as i32).collect();
- self.len_encoder.put(&lengths)?;
- for byte_array in val_it() {
- self.encoded_size += byte_array.len();
- self.data.push(byte_array.clone());
- }
-
- Ok(())
- }
-
- // Performance Note:
- // As far as can be seen these functions are rarely called and as such we can hint to the
- // compiler that they dont need to be folded into hot locations in the final output.
- #[cold]
- fn encoding(&self) -> Encoding {
- Encoding::DELTA_LENGTH_BYTE_ARRAY
- }
-
- fn estimated_data_encoded_size(&self) -> usize {
- self.len_encoder.estimated_data_encoded_size() + self.encoded_size
- }
-
- fn flush_buffer(&mut self) -> Result<ByteBufferPtr> {
- ensure_phys_ty!(
- Type::BYTE_ARRAY | Type::FIXED_LEN_BYTE_ARRAY,
- "DeltaLengthByteArrayEncoder only supports ByteArrayType"
- );
-
- let mut total_bytes = vec![];
- let lengths = self.len_encoder.flush_buffer()?;
- total_bytes.extend_from_slice(lengths.data());
- self.data.iter().for_each(|byte_array| {
- total_bytes.extend_from_slice(byte_array.data());
- });
- self.data.clear();
- self.encoded_size = 0;
-
- Ok(ByteBufferPtr::new(total_bytes))
- }
-}
-
-// ----------------------------------------------------------------------
-// DELTA_BYTE_ARRAY encoding
-
-/// Encoding for byte arrays, prefix lengths are encoded using DELTA_BINARY_PACKED
-/// encoding, followed by suffixes with DELTA_LENGTH_BYTE_ARRAY encoding.
-pub struct DeltaByteArrayEncoder<T: DataType> {
- prefix_len_encoder: DeltaBitPackEncoder<Int32Type>,
- suffix_writer: DeltaLengthByteArrayEncoder<ByteArrayType>,
- previous: Vec<u8>,
- _phantom: PhantomData<T>,
-}
-
-impl<T: DataType> DeltaByteArrayEncoder<T> {
- /// Creates new delta byte array encoder.
- pub fn new() -> Self {
- Self {
- prefix_len_encoder: DeltaBitPackEncoder::new(),
- suffix_writer: DeltaLengthByteArrayEncoder::new(),
- previous: vec![],
- _phantom: PhantomData,
- }
- }
-}
-
-impl<T: DataType> Encoder<T> for DeltaByteArrayEncoder<T> {
- fn put(&mut self, values: &[T::T]) -> Result<()> {
- let mut prefix_lengths: Vec<i32> = vec![];
- let mut suffixes: Vec<ByteArray> = vec![];
-
- let values = values.iter()
- .map(|x| x.as_any())
- .map(|x| match T::get_physical_type() {
- Type::BYTE_ARRAY => x.downcast_ref::<ByteArray>().unwrap(),
- Type::FIXED_LEN_BYTE_ARRAY => x.downcast_ref::<FixedLenByteArray>().unwrap(),
- _ => panic!(
- "DeltaByteArrayEncoder only supports ByteArrayType and FixedLenByteArrayType"
- )
- });
-
- for byte_array in values {
- let current = byte_array.data();
- // Maximum prefix length that is shared between previous value and current
- // value
- let prefix_len = cmp::min(self.previous.len(), current.len());
- let mut match_len = 0;
- while match_len < prefix_len && self.previous[match_len] == current[match_len]
- {
- match_len += 1;
- }
- prefix_lengths.push(match_len as i32);
- suffixes.push(byte_array.slice(match_len, byte_array.len() - match_len));
- // Update previous for the next prefix
- self.previous.clear();
- self.previous.extend_from_slice(current);
- }
- self.prefix_len_encoder.put(&prefix_lengths)?;
- self.suffix_writer.put(&suffixes)?;
-
- Ok(())
- }
-
- // Performance Note:
- // As far as can be seen these functions are rarely called and as such we can hint to the
- // compiler that they dont need to be folded into hot locations in the final output.
- #[cold]
- fn encoding(&self) -> Encoding {
- Encoding::DELTA_BYTE_ARRAY
- }
-
- fn estimated_data_encoded_size(&self) -> usize {
- self.prefix_len_encoder.estimated_data_encoded_size()
- + self.suffix_writer.estimated_data_encoded_size()
- }
-
- fn flush_buffer(&mut self) -> Result<ByteBufferPtr> {
- match T::get_physical_type() {
- Type::BYTE_ARRAY | Type::FIXED_LEN_BYTE_ARRAY => {
- // TODO: investigate if we can merge lengths and suffixes
- // without copying data into new vector.
- let mut total_bytes = vec![];
- // Insert lengths ...
- let lengths = self.prefix_len_encoder.flush_buffer()?;
- total_bytes.extend_from_slice(lengths.data());
- // ... followed by suffixes
- let suffixes = self.suffix_writer.flush_buffer()?;
- total_bytes.extend_from_slice(suffixes.data());
-
- self.previous.clear();
- Ok(ByteBufferPtr::new(total_bytes))
- }
- _ => panic!(
- "DeltaByteArrayEncoder only supports ByteArrayType and FixedLenByteArrayType"
- )
- }
- }
-}
-
-#[cfg(test)]
-mod tests {
- use super::*;
-
- use std::sync::Arc;
-
- use crate::decoding::{get_decoder, Decoder, DictDecoder, PlainDecoder};
- use crate::schema::types::{
- ColumnDescPtr, ColumnDescriptor, ColumnPath, Type as SchemaType,
- };
- use crate::util::{
- memory::MemTracker,
- test_common::{random_bytes, RandGen},
- };
-
- const TEST_SET_SIZE: usize = 1024;
-
- #[test]
- fn test_get_encoders() {
- // supported encodings
- create_and_check_encoder::<Int32Type>(Encoding::PLAIN, None);
- create_and_check_encoder::<Int32Type>(Encoding::DELTA_BINARY_PACKED, None);
- create_and_check_encoder::<Int32Type>(Encoding::DELTA_LENGTH_BYTE_ARRAY, None);
- create_and_check_encoder::<Int32Type>(Encoding::DELTA_BYTE_ARRAY, None);
- create_and_check_encoder::<BoolType>(Encoding::RLE, None);
-
- // error when initializing
- create_and_check_encoder::<Int32Type>(
- Encoding::RLE_DICTIONARY,
- Some(general_err!(
- "Cannot initialize this encoding through this function"
- )),
- );
- create_and_check_encoder::<Int32Type>(
- Encoding::PLAIN_DICTIONARY,
- Some(general_err!(
- "Cannot initialize this encoding through this function"
- )),
- );
-
- // unsupported
- create_and_check_encoder::<Int32Type>(
- Encoding::BIT_PACKED,
- Some(nyi_err!("Encoding BIT_PACKED is not supported")),
- );
- }
-
- #[test]
- fn test_bool() {
- BoolType::test(Encoding::PLAIN, TEST_SET_SIZE, -1);
- BoolType::test(Encoding::PLAIN_DICTIONARY, TEST_SET_SIZE, -1);
- BoolType::test(Encoding::RLE, TEST_SET_SIZE, -1);
- }
-
- #[test]
- fn test_i32() {
- Int32Type::test(Encoding::PLAIN, TEST_SET_SIZE, -1);
- Int32Type::test(Encoding::PLAIN_DICTIONARY, TEST_SET_SIZE, -1);
- Int32Type::test(Encoding::DELTA_BINARY_PACKED, TEST_SET_SIZE, -1);
- }
-
- #[test]
- fn test_i64() {
- Int64Type::test(Encoding::PLAIN, TEST_SET_SIZE, -1);
- Int64Type::test(Encoding::PLAIN_DICTIONARY, TEST_SET_SIZE, -1);
- Int64Type::test(Encoding::DELTA_BINARY_PACKED, TEST_SET_SIZE, -1);
- }
-
- #[test]
- fn test_i96() {
- Int96Type::test(Encoding::PLAIN, TEST_SET_SIZE, -1);
- Int96Type::test(Encoding::PLAIN_DICTIONARY, TEST_SET_SIZE, -1);
- }
-
- #[test]
- fn test_float() {
- FloatType::test(Encoding::PLAIN, TEST_SET_SIZE, -1);
- FloatType::test(Encoding::PLAIN_DICTIONARY, TEST_SET_SIZE, -1);
- }
-
- #[test]
- fn test_double() {
- DoubleType::test(Encoding::PLAIN, TEST_SET_SIZE, -1);
- DoubleType::test(Encoding::PLAIN_DICTIONARY, TEST_SET_SIZE, -1);
- }
-
- #[test]
- fn test_byte_array() {
- ByteArrayType::test(Encoding::PLAIN, TEST_SET_SIZE, -1);
- ByteArrayType::test(Encoding::PLAIN_DICTIONARY, TEST_SET_SIZE, -1);
- ByteArrayType::test(Encoding::DELTA_LENGTH_BYTE_ARRAY, TEST_SET_SIZE, -1);
- ByteArrayType::test(Encoding::DELTA_BYTE_ARRAY, TEST_SET_SIZE, -1);
- }
-
- #[test]
- fn test_fixed_lenbyte_array() {
- FixedLenByteArrayType::test(Encoding::PLAIN, TEST_SET_SIZE, 100);
- FixedLenByteArrayType::test(Encoding::PLAIN_DICTIONARY, TEST_SET_SIZE, 100);
- FixedLenByteArrayType::test(Encoding::DELTA_BYTE_ARRAY, TEST_SET_SIZE, 100);
- }
-
- #[test]
- fn test_dict_encoded_size() {
- fn run_test<T: DataType>(
- type_length: i32,
- values: &[T::T],
- expected_size: usize,
- ) {
- let mut encoder = create_test_dict_encoder::<T>(type_length);
- assert_eq!(encoder.dict_encoded_size(), 0);
- encoder.put(values).unwrap();
- assert_eq!(encoder.dict_encoded_size(), expected_size);
- // We do not reset encoded size of the dictionary keys after flush_buffer
- encoder.flush_buffer().unwrap();
- assert_eq!(encoder.dict_encoded_size(), expected_size);
- }
-
- // Only 2 variations of values 1 byte each
- run_test::<BoolType>(-1, &[true, false, true, false, true], 2);
- run_test::<Int32Type>(-1, &[1i32, 2i32, 3i32, 4i32, 5i32], 20);
- run_test::<Int64Type>(-1, &[1i64, 2i64, 3i64, 4i64, 5i64], 40);
- run_test::<FloatType>(-1, &[1f32, 2f32, 3f32, 4f32, 5f32], 20);
- run_test::<DoubleType>(-1, &[1f64, 2f64, 3f64, 4f64, 5f64], 40);
- // Int96: len + reference
- run_test::<Int96Type>(
- -1,
- &[Int96::from(vec![1, 2, 3]), Int96::from(vec![2, 3, 4])],
- 32,
- );
- run_test::<ByteArrayType>(
- -1,
- &[ByteArray::from("abcd"), ByteArray::from("efj")],
- 15,
- );
- run_test::<FixedLenByteArrayType>(
- 2,
- &[ByteArray::from("ab").into(), ByteArray::from("bc").into()],
- 4,
- );
- }
-
- #[test]
- fn test_estimated_data_encoded_size() {
- fn run_test<T: DataType>(
- encoding: Encoding,
- type_length: i32,
- values: &[T::T],
- initial_size: usize,
- max_size: usize,
- flush_size: usize,
- ) {
- let mut encoder = match encoding {
- Encoding::PLAIN_DICTIONARY | Encoding::RLE_DICTIONARY => {
- Box::new(create_test_dict_encoder::<T>(type_length))
- }
- _ => create_test_encoder::<T>(type_length, encoding),
- };
- assert_eq!(encoder.estimated_data_encoded_size(), initial_size);
-
- encoder.put(values).unwrap();
- assert_eq!(encoder.estimated_data_encoded_size(), max_size);
-
- encoder.flush_buffer().unwrap();
- assert_eq!(encoder.estimated_data_encoded_size(), flush_size);
- }
-
- // PLAIN
- run_test::<Int32Type>(Encoding::PLAIN, -1, &[123; 1024], 0, 4096, 0);
-
- // DICTIONARY
- // NOTE: The final size is almost the same because the dictionary entries are
- // preserved after encoded values have been written.
- run_test::<Int32Type>(Encoding::RLE_DICTIONARY, -1, &[123, 1024], 11, 68, 66);
-
- // DELTA_BINARY_PACKED
- run_test::<Int32Type>(Encoding::DELTA_BINARY_PACKED, -1, &[123; 1024], 0, 35, 0);
-
- // RLE
- let mut values = vec![];
- values.extend_from_slice(&[true; 16]);
- values.extend_from_slice(&[false; 16]);
- run_test::<BoolType>(Encoding::RLE, -1, &values, 0, 2, 0);
-
- // DELTA_LENGTH_BYTE_ARRAY
- run_test::<ByteArrayType>(
- Encoding::DELTA_LENGTH_BYTE_ARRAY,
- -1,
- &[ByteArray::from("ab"), ByteArray::from("abc")],
- 0,
- 5, // only value bytes, length encoder is not flushed yet
- 0,
- );
-
- // DELTA_BYTE_ARRAY
- run_test::<ByteArrayType>(
- Encoding::DELTA_BYTE_ARRAY,
- -1,
- &[ByteArray::from("ab"), ByteArray::from("abc")],
- 0,
- 3, // only suffix bytes, length encoder is not flushed yet
- 0,
- );
- }
-
- // See: https://github.com/sunchao/parquet-rs/issues/47
- #[test]
- fn test_issue_47() {
- let mut encoder =
- create_test_encoder::<ByteArrayType>(0, Encoding::DELTA_BYTE_ARRAY);
- let mut decoder =
- create_test_decoder::<ByteArrayType>(0, Encoding::DELTA_BYTE_ARRAY);
-
- let mut input = vec![];
- input.push(ByteArray::from("aa"));
- input.push(ByteArray::from("aaa"));
- input.push(ByteArray::from("aa"));
- input.push(ByteArray::from("aaa"));
- let mut output = vec![ByteArray::default(); input.len()];
-
- let mut result =
- put_and_get(&mut encoder, &mut decoder, &input[..2], &mut output[..2]);
- assert!(
- result.is_ok(),
- "first put_and_get() failed with: {}",
- result.unwrap_err()
- );
- result = put_and_get(&mut encoder, &mut decoder, &input[2..], &mut output[2..]);
- assert!(
- result.is_ok(),
- "second put_and_get() failed with: {}",
- result.unwrap_err()
- );
- assert_eq!(output, input);
- }
-
- trait EncodingTester<T: DataType> {
- fn test(enc: Encoding, total: usize, type_length: i32) {
- let result = match enc {
- Encoding::PLAIN_DICTIONARY | Encoding::RLE_DICTIONARY => {
- Self::test_dict_internal(total, type_length)
- }
- enc => Self::test_internal(enc, total, type_length),
- };
-
- assert!(
- result.is_ok(),
- "Expected result to be OK but got err:\n {}",
- result.unwrap_err()
- );
- }
-
- fn test_internal(enc: Encoding, total: usize, type_length: i32) -> Result<()>;
-
- fn test_dict_internal(total: usize, type_length: i32) -> Result<()>;
- }
-
- impl<T: DataType + RandGen<T>> EncodingTester<T> for T {
- fn test_internal(enc: Encoding, total: usize, type_length: i32) -> Result<()> {
- let mut encoder = create_test_encoder::<T>(type_length, enc);
- let mut decoder = create_test_decoder::<T>(type_length, enc);
- let mut values = <T as RandGen<T>>::gen_vec(type_length, total);
- let mut result_data = vec![T::T::default(); total];
-
- // Test put/get spaced.
- let num_bytes = bit_util::ceil(total as i64, 8);
- let valid_bits = random_bytes(num_bytes as usize);
- let values_written = encoder.put_spaced(&values[..], &valid_bits[..])?;
- let data = encoder.flush_buffer()?;
- decoder.set_data(data, values_written)?;
- let _ = decoder.get_spaced(
- &mut result_data[..],
- values.len() - values_written,
- &valid_bits[..],
- )?;
-
- // Check equality
- for i in 0..total {
- if bit_util::get_bit(&valid_bits[..], i) {
- assert_eq!(result_data[i], values[i]);
- } else {
- assert_eq!(result_data[i], T::T::default());
- }
- }
-
- let mut actual_total = put_and_get(
- &mut encoder,
- &mut decoder,
- &values[..],
- &mut result_data[..],
- )?;
- assert_eq!(actual_total, total);
- assert_eq!(result_data, values);
-
- // Encode more data after flush and test with decoder
-
- values = <T as RandGen<T>>::gen_vec(type_length, total);
- actual_total = put_and_get(
- &mut encoder,
- &mut decoder,
- &values[..],
- &mut result_data[..],
- )?;
- assert_eq!(actual_total, total);
- assert_eq!(result_data, values);
-
- Ok(())
- }
-
- fn test_dict_internal(total: usize, type_length: i32) -> Result<()> {
- let mut encoder = create_test_dict_encoder::<T>(type_length);
- let mut values = <T as RandGen<T>>::gen_vec(type_length, total);
- encoder.put(&values[..])?;
-
- let mut data = encoder.flush_buffer()?;
- let mut decoder = create_test_dict_decoder::<T>();
- let mut dict_decoder = PlainDecoder::<T>::new(type_length);
- dict_decoder.set_data(encoder.write_dict()?, encoder.num_entries())?;
- decoder.set_dict(Box::new(dict_decoder))?;
- let mut result_data = vec![T::T::default(); total];
- decoder.set_data(data, total)?;
- let mut actual_total = decoder.get(&mut result_data)?;
-
- assert_eq!(actual_total, total);
- assert_eq!(result_data, values);
-
- // Encode more data after flush and test with decoder
-
- values = <T as RandGen<T>>::gen_vec(type_length, total);
- encoder.put(&values[..])?;
- data = encoder.flush_buffer()?;
-
- let mut dict_decoder = PlainDecoder::<T>::new(type_length);
- dict_decoder.set_data(encoder.write_dict()?, encoder.num_entries())?;
- decoder.set_dict(Box::new(dict_decoder))?;
- decoder.set_data(data, total)?;
- actual_total = decoder.get(&mut result_data)?;
-
- assert_eq!(actual_total, total);
- assert_eq!(result_data, values);
-
- Ok(())
- }
- }
-
- fn put_and_get<T: DataType>(
- encoder: &mut Box<dyn Encoder<T>>,
- decoder: &mut Box<dyn Decoder<T>>,
- input: &[T::T],
- output: &mut [T::T],
- ) -> Result<usize> {
- encoder.put(input)?;
- let data = encoder.flush_buffer()?;
- decoder.set_data(data, input.len())?;
- decoder.get(output)
- }
-
- fn create_and_check_encoder<T: DataType>(
- encoding: Encoding,
- err: Option<ParquetError>,
- ) {
- let descr = create_test_col_desc_ptr(-1, T::get_physical_type());
- let mem_tracker = Arc::new(MemTracker::new());
- let encoder = get_encoder::<T>(descr, encoding, mem_tracker);
- match err {
- Some(parquet_error) => {
- assert!(encoder.is_err());
- assert_eq!(encoder.err().unwrap(), parquet_error);
- }
- None => {
- assert!(encoder.is_ok());
- assert_eq!(encoder.unwrap().encoding(), encoding);
- }
- }
- }
-
- // Creates test column descriptor.
- fn create_test_col_desc_ptr(type_len: i32, t: Type) -> ColumnDescPtr {
- let ty = SchemaType::primitive_type_builder("t", t)
- .with_length(type_len)
- .build()
- .unwrap();
- Arc::new(ColumnDescriptor::new(
- Arc::new(ty),
- 0,
- 0,
- ColumnPath::new(vec![]),
- ))
- }
-
- fn create_test_encoder<T: DataType>(
- type_len: i32,
- enc: Encoding,
- ) -> Box<dyn Encoder<T>> {
- let desc = create_test_col_desc_ptr(type_len, T::get_physical_type());
- let mem_tracker = Arc::new(MemTracker::new());
- get_encoder(desc, enc, mem_tracker).unwrap()
- }
-
- fn create_test_decoder<T: DataType>(
- type_len: i32,
- enc: Encoding,
- ) -> Box<dyn Decoder<T>> {
- let desc = create_test_col_desc_ptr(type_len, T::get_physical_type());
- get_decoder(desc, enc).unwrap()
- }
-
- fn create_test_dict_encoder<T: DataType>(type_len: i32) -> DictEncoder<T> {
- let desc = create_test_col_desc_ptr(type_len, T::get_physical_type());
- let mem_tracker = Arc::new(MemTracker::new());
- DictEncoder::<T>::new(desc, mem_tracker)
- }
-
- fn create_test_dict_decoder<T: DataType>() -> DictDecoder<T> {
- DictDecoder::<T>::new()
- }
-}
diff --git a/parquet/src/encodings/levels.rs b/parquet/src/encodings/levels.rs
deleted file mode 100644
index 6727589..0000000
--- a/parquet/src/encodings/levels.rs
+++ /dev/null
@@ -1,563 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-use std::{cmp, mem};
-
-use super::rle::{RleDecoder, RleEncoder};
-
-use crate::basic::Encoding;
-use crate::data_type::AsBytes;
-use crate::errors::{ParquetError, Result};
-use crate::util::{
- bit_util::{ceil, log2, BitReader, BitWriter},
- memory::ByteBufferPtr,
-};
-
-/// Computes max buffer size for level encoder/decoder based on encoding, max
-/// repetition/definition level and number of total buffered values (includes null
-/// values).
-#[inline]
-pub fn max_buffer_size(
- encoding: Encoding,
- max_level: i16,
- num_buffered_values: usize,
-) -> usize {
- let bit_width = log2(max_level as u64 + 1) as u8;
- match encoding {
- Encoding::RLE => {
- RleEncoder::max_buffer_size(bit_width, num_buffered_values)
- + RleEncoder::min_buffer_size(bit_width)
- }
- Encoding::BIT_PACKED => {
- ceil((num_buffered_values * bit_width as usize) as i64, 8) as usize
- }
- _ => panic!("Unsupported encoding type {}", encoding),
- }
-}
-
-/// Encoder for definition/repetition levels.
-/// Currently only supports RLE and BIT_PACKED (dev/null) encoding, including v2.
-pub enum LevelEncoder {
- RLE(RleEncoder),
- RLE_V2(RleEncoder),
- BIT_PACKED(u8, BitWriter),
-}
-
-impl LevelEncoder {
- /// Creates new level encoder based on encoding, max level and underlying byte buffer.
- /// For bit packed encoding it is assumed that buffer is already allocated with
- /// `levels::max_buffer_size` method.
- ///
- /// Used to encode levels for Data Page v1.
- ///
- /// Panics, if encoding is not supported.
- pub fn v1(encoding: Encoding, max_level: i16, byte_buffer: Vec<u8>) -> Self {
- let bit_width = log2(max_level as u64 + 1) as u8;
- match encoding {
- Encoding::RLE => LevelEncoder::RLE(RleEncoder::new_from_buf(
- bit_width,
- byte_buffer,
- mem::size_of::<i32>(),
- )),
- Encoding::BIT_PACKED => {
- // Here we set full byte buffer without adjusting for num_buffered_values,
- // because byte buffer will already be allocated with size from
- // `max_buffer_size()` method.
- LevelEncoder::BIT_PACKED(
- bit_width,
- BitWriter::new_from_buf(byte_buffer, 0),
- )
- }
- _ => panic!("Unsupported encoding type {}", encoding),
- }
- }
-
- /// Creates new level encoder based on RLE encoding. Used to encode Data Page v2
- /// repetition and definition levels.
- pub fn v2(max_level: i16, byte_buffer: Vec<u8>) -> Self {
- let bit_width = log2(max_level as u64 + 1) as u8;
- LevelEncoder::RLE_V2(RleEncoder::new_from_buf(bit_width, byte_buffer, 0))
- }
-
- /// Put/encode levels vector into this level encoder.
- /// Returns number of encoded values that are less than or equal to length of the
- /// input buffer.
- ///
- /// RLE and BIT_PACKED level encoders return Err() when internal buffer overflows or
- /// flush fails.
- #[inline]
- pub fn put(&mut self, buffer: &[i16]) -> Result<usize> {
- let mut num_encoded = 0;
- match *self {
- LevelEncoder::RLE(ref mut encoder)
- | LevelEncoder::RLE_V2(ref mut encoder) => {
- for value in buffer {
- if !encoder.put(*value as u64)? {
- return Err(general_err!("RLE buffer is full"));
- }
- num_encoded += 1;
- }
- encoder.flush()?;
- }
- LevelEncoder::BIT_PACKED(bit_width, ref mut encoder) => {
- for value in buffer {
- if !encoder.put_value(*value as u64, bit_width as usize) {
- return Err(general_err!("Not enough bytes left"));
- }
- num_encoded += 1;
- }
- encoder.flush();
- }
- }
- Ok(num_encoded)
- }
-
- /// Finalizes level encoder, flush all intermediate buffers and return resulting
- /// encoded buffer. Returned buffer is already truncated to encoded bytes only.
- #[inline]
- pub fn consume(self) -> Result<Vec<u8>> {
- match self {
- LevelEncoder::RLE(encoder) => {
- let mut encoded_data = encoder.consume()?;
- // Account for the buffer offset
- let encoded_len = encoded_data.len() - mem::size_of::<i32>();
- let len = (encoded_len as i32).to_le();
- let len_bytes = len.as_bytes();
- encoded_data[0..len_bytes.len()].copy_from_slice(len_bytes);
- Ok(encoded_data)
- }
- LevelEncoder::RLE_V2(encoder) => encoder.consume(),
- LevelEncoder::BIT_PACKED(_, encoder) => Ok(encoder.consume()),
- }
- }
-}
-
-/// Decoder for definition/repetition levels.
-/// Currently only supports RLE and BIT_PACKED encoding for Data Page v1 and
-/// RLE for Data Page v2.
-pub enum LevelDecoder {
- RLE(Option<usize>, RleDecoder),
- RLE_V2(Option<usize>, RleDecoder),
- BIT_PACKED(Option<usize>, u8, BitReader),
-}
-
-impl LevelDecoder {
- /// Creates new level decoder based on encoding and max definition/repetition level.
- /// This method only initializes level decoder, `set_data` method must be called
- /// before reading any value.
- ///
- /// Used to encode levels for Data Page v1.
- ///
- /// Panics if encoding is not supported
- pub fn v1(encoding: Encoding, max_level: i16) -> Self {
- let bit_width = log2(max_level as u64 + 1) as u8;
- match encoding {
- Encoding::RLE => LevelDecoder::RLE(None, RleDecoder::new(bit_width)),
- Encoding::BIT_PACKED => {
- LevelDecoder::BIT_PACKED(None, bit_width, BitReader::from(Vec::new()))
- }
- _ => panic!("Unsupported encoding type {}", encoding),
- }
- }
-
- /// Creates new level decoder based on RLE encoding.
- /// Used to decode Data Page v2 repetition and definition levels.
- ///
- /// To set data for this decoder, use `set_data_range` method.
- pub fn v2(max_level: i16) -> Self {
- let bit_width = log2(max_level as u64 + 1) as u8;
- LevelDecoder::RLE_V2(None, RleDecoder::new(bit_width))
- }
-
- /// Sets data for this level decoder, and returns total number of bytes set.
- /// This is used for Data Page v1 levels.
- ///
- /// `data` is encoded data as byte buffer, `num_buffered_values` represents total
- /// number of values that is expected.
- ///
- /// Both RLE and BIT_PACKED level decoders set `num_buffered_values` as total number
- /// of values that they can return and track num values.
- #[inline]
- pub fn set_data(&mut self, num_buffered_values: usize, data: ByteBufferPtr) -> usize {
- match *self {
- LevelDecoder::RLE(ref mut num_values, ref mut decoder) => {
- *num_values = Some(num_buffered_values);
- let i32_size = mem::size_of::<i32>();
- let data_size = read_num_bytes!(i32, i32_size, data.as_ref()) as usize;
- decoder.set_data(data.range(i32_size, data_size));
- i32_size + data_size
- }
- LevelDecoder::BIT_PACKED(ref mut num_values, bit_width, ref mut decoder) => {
- *num_values = Some(num_buffered_values);
- // Set appropriate number of bytes: if max size is larger than buffer -
- // set full buffer
- let num_bytes =
- ceil((num_buffered_values * bit_width as usize) as i64, 8);
- let data_size = cmp::min(num_bytes as usize, data.len());
- decoder.reset(data.range(data.start(), data_size));
- data_size
- }
- _ => panic!(),
- }
- }
-
- /// Sets byte array explicitly when start position `start` and length `len` are known
- /// in advance. Only supported by RLE level decoder and used for Data Page v2 levels.
- /// Returns number of total bytes set for this decoder (len).
- #[inline]
- pub fn set_data_range(
- &mut self,
- num_buffered_values: usize,
- data: &ByteBufferPtr,
- start: usize,
- len: usize,
- ) -> usize {
- match *self {
- LevelDecoder::RLE_V2(ref mut num_values, ref mut decoder) => {
- decoder.set_data(data.range(start, len));
- *num_values = Some(num_buffered_values);
- len
- }
- _ => panic!(
- "set_data_range() method is only supported by RLE v2 encoding type"
- ),
- }
- }
-
- /// Returns true if data is set for decoder, false otherwise.
- #[inline]
- pub fn is_data_set(&self) -> bool {
- match self {
- LevelDecoder::RLE(ref num_values, _) => num_values.is_some(),
- LevelDecoder::RLE_V2(ref num_values, _) => num_values.is_some(),
- LevelDecoder::BIT_PACKED(ref num_values, ..) => num_values.is_some(),
- }
- }
-
- /// Decodes values and puts them into `buffer`.
- /// Returns number of values that were successfully decoded (less than or equal to
- /// buffer length).
- #[inline]
- pub fn get(&mut self, buffer: &mut [i16]) -> Result<usize> {
- assert!(self.is_data_set(), "No data set for decoding");
- match *self {
- LevelDecoder::RLE(ref mut num_values, ref mut decoder)
- | LevelDecoder::RLE_V2(ref mut num_values, ref mut decoder) => {
- // Max length we can read
- let len = cmp::min(num_values.unwrap(), buffer.len());
- let values_read = decoder.get_batch::<i16>(&mut buffer[0..len])?;
- *num_values = num_values.map(|len| len - values_read);
- Ok(values_read)
- }
- LevelDecoder::BIT_PACKED(ref mut num_values, bit_width, ref mut decoder) => {
- // When extracting values from bit reader, it might return more values
- // than left because of padding to a full byte, we use
- // num_values to track precise number of values.
- let len = cmp::min(num_values.unwrap(), buffer.len());
- let values_read =
- decoder.get_batch::<i16>(&mut buffer[..len], bit_width as usize);
- *num_values = num_values.map(|len| len - values_read);
- Ok(values_read)
- }
- }
- }
-}
-
-#[cfg(test)]
-mod tests {
- use super::*;
-
- use crate::util::test_common::random_numbers_range;
-
- fn test_internal_roundtrip(enc: Encoding, levels: &[i16], max_level: i16, v2: bool) {
- let size = max_buffer_size(enc, max_level, levels.len());
- let mut encoder = if v2 {
- LevelEncoder::v2(max_level, vec![0; size])
- } else {
- LevelEncoder::v1(enc, max_level, vec![0; size])
- };
- encoder.put(&levels).expect("put() should be OK");
- let encoded_levels = encoder.consume().expect("consume() should be OK");
-
- let byte_buf = ByteBufferPtr::new(encoded_levels);
- let mut decoder;
- if v2 {
- decoder = LevelDecoder::v2(max_level);
- decoder.set_data_range(levels.len(), &byte_buf, 0, byte_buf.len());
- } else {
- decoder = LevelDecoder::v1(enc, max_level);
- decoder.set_data(levels.len(), byte_buf);
- };
-
- let mut buffer = vec![0; levels.len()];
- let num_decoded = decoder.get(&mut buffer).expect("get() should be OK");
- assert_eq!(num_decoded, levels.len());
- assert_eq!(buffer, levels);
- }
-
- // Performs incremental read until all bytes are read
- fn test_internal_roundtrip_incremental(
- enc: Encoding,
- levels: &[i16],
- max_level: i16,
- v2: bool,
- ) {
- let size = max_buffer_size(enc, max_level, levels.len());
- let mut encoder = if v2 {
- LevelEncoder::v2(max_level, vec![0; size])
- } else {
- LevelEncoder::v1(enc, max_level, vec![0; size])
- };
- encoder.put(&levels).expect("put() should be OK");
- let encoded_levels = encoder.consume().expect("consume() should be OK");
-
- let byte_buf = ByteBufferPtr::new(encoded_levels);
- let mut decoder;
- if v2 {
- decoder = LevelDecoder::v2(max_level);
- decoder.set_data_range(levels.len(), &byte_buf, 0, byte_buf.len());
- } else {
- decoder = LevelDecoder::v1(enc, max_level);
- decoder.set_data(levels.len(), byte_buf);
- }
-
- let mut buffer = vec![0; levels.len() * 2];
- let mut total_decoded = 0;
- let mut safe_stop = levels.len() * 2; // still terminate in case of issues in the code
- while safe_stop > 0 {
- safe_stop -= 1;
- let num_decoded = decoder
- .get(&mut buffer[total_decoded..total_decoded + 1])
- .expect("get() should be OK");
- if num_decoded == 0 {
- break;
- }
- total_decoded += num_decoded;
- }
- assert!(
- safe_stop > 0,
- "Failed to read values incrementally, reached safe stop"
- );
- assert_eq!(total_decoded, levels.len());
- assert_eq!(&buffer[0..levels.len()], levels);
- }
-
- // Tests encoding/decoding of values when output buffer is larger than number of
- // encoded values
- fn test_internal_roundtrip_underflow(
- enc: Encoding,
- levels: &[i16],
- max_level: i16,
- v2: bool,
- ) {
- let size = max_buffer_size(enc, max_level, levels.len());
- let mut encoder = if v2 {
- LevelEncoder::v2(max_level, vec![0; size])
- } else {
- LevelEncoder::v1(enc, max_level, vec![0; size])
- };
- // Encode only one value
- let num_encoded = encoder.put(&levels[0..1]).expect("put() should be OK");
- let encoded_levels = encoder.consume().expect("consume() should be OK");
- assert_eq!(num_encoded, 1);
-
- let byte_buf = ByteBufferPtr::new(encoded_levels);
- let mut decoder;
- // Set one encoded value as `num_buffered_values`
- if v2 {
- decoder = LevelDecoder::v2(max_level);
- decoder.set_data_range(1, &byte_buf, 0, byte_buf.len());
- } else {
- decoder = LevelDecoder::v1(enc, max_level);
- decoder.set_data(1, byte_buf);
- }
-
- let mut buffer = vec![0; levels.len()];
- let num_decoded = decoder.get(&mut buffer).expect("get() should be OK");
- assert_eq!(num_decoded, num_encoded);
- assert_eq!(buffer[0..num_decoded], levels[0..num_decoded]);
- }
-
- // Tests when encoded values are larger than encoder's buffer
- fn test_internal_roundtrip_overflow(
- enc: Encoding,
- levels: &[i16],
- max_level: i16,
- v2: bool,
- ) {
- let size = max_buffer_size(enc, max_level, levels.len());
- let mut encoder = if v2 {
- LevelEncoder::v2(max_level, vec![0; size])
- } else {
- LevelEncoder::v1(enc, max_level, vec![0; size])
- };
- let mut found_err = false;
- // Insert a large number of values, so we run out of space
- for _ in 0..100 {
- if let Err(err) = encoder.put(&levels) {
- assert!(format!("{}", err).contains("Not enough bytes left"));
- found_err = true;
- break;
- };
- }
- if !found_err {
- panic!("Failed test: no buffer overflow");
- }
- }
-
- #[test]
- fn test_roundtrip_one() {
- let levels = vec![0, 1, 1, 1, 1, 0, 0, 0, 0, 1];
- let max_level = 1;
- test_internal_roundtrip(Encoding::RLE, &levels, max_level, false);
- test_internal_roundtrip(Encoding::BIT_PACKED, &levels, max_level, false);
- test_internal_roundtrip(Encoding::RLE, &levels, max_level, true);
- }
-
- #[test]
- fn test_roundtrip() {
- let levels = vec![0, 1, 2, 3, 4, 5, 6, 7, 8, 9];
- let max_level = 10;
- test_internal_roundtrip(Encoding::RLE, &levels, max_level, false);
- test_internal_roundtrip(Encoding::BIT_PACKED, &levels, max_level, false);
- test_internal_roundtrip(Encoding::RLE, &levels, max_level, true);
- }
-
- #[test]
- fn test_roundtrip_incremental() {
- let levels = vec![0, 1, 2, 3, 4, 5, 6, 7, 8, 9];
- let max_level = 10;
- test_internal_roundtrip_incremental(Encoding::RLE, &levels, max_level, false);
- test_internal_roundtrip_incremental(
- Encoding::BIT_PACKED,
- &levels,
- max_level,
- false,
- );
- test_internal_roundtrip_incremental(Encoding::RLE, &levels, max_level, true);
- }
-
- #[test]
- fn test_roundtrip_all_zeros() {
- let levels = vec![0, 0, 0, 0, 0, 0, 0, 0, 0, 0];
- let max_level = 1;
- test_internal_roundtrip(Encoding::RLE, &levels, max_level, false);
- test_internal_roundtrip(Encoding::BIT_PACKED, &levels, max_level, false);
- test_internal_roundtrip(Encoding::RLE, &levels, max_level, true);
- }
-
- #[test]
- fn test_roundtrip_random() {
- // This test is mainly for bit packed level encoder/decoder
- let mut levels = Vec::new();
- let max_level = 5;
- random_numbers_range::<i16>(120, 0, max_level, &mut levels);
- test_internal_roundtrip(Encoding::RLE, &levels, max_level, false);
- test_internal_roundtrip(Encoding::BIT_PACKED, &levels, max_level, false);
- test_internal_roundtrip(Encoding::RLE, &levels, max_level, true);
- }
-
- #[test]
- fn test_roundtrip_underflow() {
- let levels = vec![1, 1, 2, 3, 2, 1, 1, 2, 3, 1];
- let max_level = 3;
- test_internal_roundtrip_underflow(Encoding::RLE, &levels, max_level, false);
- test_internal_roundtrip_underflow(
- Encoding::BIT_PACKED,
- &levels,
- max_level,
- false,
- );
- test_internal_roundtrip_underflow(Encoding::RLE, &levels, max_level, true);
- }
-
- #[test]
- fn test_roundtrip_overflow() {
- let levels = vec![1, 1, 2, 3, 2, 1, 1, 2, 3, 1];
- let max_level = 3;
- test_internal_roundtrip_overflow(Encoding::RLE, &levels, max_level, false);
- test_internal_roundtrip_overflow(Encoding::BIT_PACKED, &levels, max_level, false);
- test_internal_roundtrip_overflow(Encoding::RLE, &levels, max_level, true);
- }
-
- #[test]
- fn test_rle_decoder_set_data_range() {
- // Buffer containing both repetition and definition levels
- let buffer = ByteBufferPtr::new(vec![5, 198, 2, 5, 42, 168, 10, 0, 2, 3, 36, 73]);
-
- let max_rep_level = 1;
- let mut decoder = LevelDecoder::v2(max_rep_level);
- assert_eq!(decoder.set_data_range(10, &buffer, 0, 3), 3);
- let mut result = vec![0; 10];
- let num_decoded = decoder.get(&mut result).expect("get() should be OK");
- assert_eq!(num_decoded, 10);
- assert_eq!(result, vec![0, 1, 1, 0, 0, 0, 1, 1, 0, 1]);
-
- let max_def_level = 2;
- let mut decoder = LevelDecoder::v2(max_def_level);
- assert_eq!(decoder.set_data_range(10, &buffer, 3, 5), 5);
- let mut result = vec![0; 10];
- let num_decoded = decoder.get(&mut result).expect("get() should be OK");
- assert_eq!(num_decoded, 10);
- assert_eq!(result, vec![2, 2, 2, 0, 0, 2, 2, 2, 2, 2]);
- }
-
- #[test]
- #[should_panic(
- expected = "set_data_range() method is only supported by RLE v2 encoding type"
- )]
- fn test_bit_packed_decoder_set_data_range() {
- // Buffer containing both repetition and definition levels
- let buffer = ByteBufferPtr::new(vec![1, 2, 3, 4, 5]);
- let max_level = 1;
- let mut decoder = LevelDecoder::v1(Encoding::BIT_PACKED, max_level);
- decoder.set_data_range(10, &buffer, 0, 3);
- }
-
- #[test]
- fn test_bit_packed_decoder_set_data() {
- // Test the maximum size that is assigned based on number of values and buffer
- // length
- let buffer = ByteBufferPtr::new(vec![1, 2, 3, 4, 5]);
- let max_level = 1;
- let mut decoder = LevelDecoder::v1(Encoding::BIT_PACKED, max_level);
- // This should reset to entire buffer
- assert_eq!(decoder.set_data(1024, buffer.all()), buffer.len());
- // This should set smallest num bytes
- assert_eq!(decoder.set_data(3, buffer.all()), 1);
- }
-
- #[test]
- #[should_panic(expected = "No data set for decoding")]
- fn test_rle_level_decoder_get_no_set_data() {
- // `get()` normally panics because bit_reader is not set for RLE decoding
- // we have explicit check now in set_data
- let max_rep_level = 2;
- let mut decoder = LevelDecoder::v1(Encoding::RLE, max_rep_level);
- let mut buffer = vec![0; 16];
- decoder.get(&mut buffer).unwrap();
- }
-
- #[test]
- #[should_panic(expected = "No data set for decoding")]
- fn test_bit_packed_level_decoder_get_no_set_data() {
- let max_rep_level = 2;
- let mut decoder = LevelDecoder::v1(Encoding::BIT_PACKED, max_rep_level);
- let mut buffer = vec![0; 16];
- decoder.get(&mut buffer).unwrap();
- }
-}
diff --git a/parquet/src/encodings/mod.rs b/parquet/src/encodings/mod.rs
deleted file mode 100644
index 6046dda..0000000
--- a/parquet/src/encodings/mod.rs
+++ /dev/null
@@ -1,21 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-pub mod decoding;
-pub mod encoding;
-pub mod levels;
-pub(crate) mod rle;
diff --git a/parquet/src/encodings/rle.rs b/parquet/src/encodings/rle.rs
deleted file mode 100644
index b2a23da..0000000
--- a/parquet/src/encodings/rle.rs
+++ /dev/null
@@ -1,831 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-use std::{cmp, mem::size_of};
-
-use crate::errors::{ParquetError, Result};
-use crate::util::{
- bit_util::{self, from_ne_slice, BitReader, BitWriter, FromBytes},
- memory::ByteBufferPtr,
-};
-
-/// Rle/Bit-Packing Hybrid Encoding
-/// The grammar for this encoding looks like the following (copied verbatim
-/// from <https://github.com/Parquet/parquet-format/blob/master/Encodings.md>):
-///
-/// rle-bit-packed-hybrid: <length> <encoded-data>
-/// length := length of the <encoded-data> in bytes stored as 4 bytes little endian
-/// encoded-data := <run>*
-/// run := <bit-packed-run> | <rle-run>
-/// bit-packed-run := <bit-packed-header> <bit-packed-values>
-/// bit-packed-header := varint-encode(<bit-pack-count> << 1 | 1)
-/// we always bit-pack a multiple of 8 values at a time, so we only store the number of
-/// values / 8
-/// bit-pack-count := (number of values in this run) / 8
-/// bit-packed-values := *see 1 below*
-/// rle-run := <rle-header> <repeated-value>
-/// rle-header := varint-encode( (number of times repeated) << 1)
-/// repeated-value := value that is repeated, using a fixed-width of
-/// round-up-to-next-byte(bit-width)
-
-/// Maximum groups per bit-packed run. Current value is 64.
-const MAX_GROUPS_PER_BIT_PACKED_RUN: usize = 1 << 6;
-const MAX_VALUES_PER_BIT_PACKED_RUN: usize = MAX_GROUPS_PER_BIT_PACKED_RUN * 8;
-const MAX_WRITER_BUF_SIZE: usize = 1 << 10;
-
-/// A RLE/Bit-Packing hybrid encoder.
-// TODO: tracking memory usage
-pub struct RleEncoder {
- // Number of bits needed to encode the value. Must be in the range of [0, 64].
- bit_width: u8,
-
- // Underlying writer which holds an internal buffer.
- bit_writer: BitWriter,
-
- // The maximum byte size a single run can take.
- max_run_byte_size: usize,
-
- // Buffered values for bit-packed runs.
- buffered_values: [u64; 8],
-
- // Number of current buffered values. Must be less than 8.
- num_buffered_values: usize,
-
- // The current (also last) value that was written and the count of how many
- // times in a row that value has been seen.
- current_value: u64,
-
- // The number of repetitions for `current_value`. If this gets too high we'd
- // switch to use RLE encoding.
- repeat_count: usize,
-
- // Number of bit-packed values in the current run. This doesn't include values
- // in `buffered_values`.
- bit_packed_count: usize,
-
- // The position of the indicator byte in the `bit_writer`.
- indicator_byte_pos: i64,
-}
-
-impl RleEncoder {
- pub fn new(bit_width: u8, buffer_len: usize) -> Self {
- let buffer = vec![0; buffer_len];
- RleEncoder::new_from_buf(bit_width, buffer, 0)
- }
-
- /// Initialize the encoder from existing `buffer` and the starting offset `start`.
- pub fn new_from_buf(bit_width: u8, buffer: Vec<u8>, start: usize) -> Self {
- assert!(bit_width <= 64, "bit_width ({}) out of range.", bit_width);
- let max_run_byte_size = RleEncoder::min_buffer_size(bit_width);
- assert!(
- buffer.len() >= max_run_byte_size,
- "buffer length {} must be greater than {}",
- buffer.len(),
- max_run_byte_size
- );
- let bit_writer = BitWriter::new_from_buf(buffer, start);
- RleEncoder {
- bit_width,
- bit_writer,
- max_run_byte_size,
- buffered_values: [0; 8],
- num_buffered_values: 0,
- current_value: 0,
- repeat_count: 0,
- bit_packed_count: 0,
- indicator_byte_pos: -1,
- }
- }
-
- /// Returns the minimum buffer size needed to use the encoder for `bit_width`.
- /// This is the maximum length of a single run for `bit_width`.
- pub fn min_buffer_size(bit_width: u8) -> usize {
- let max_bit_packed_run_size = 1 + bit_util::ceil(
- (MAX_VALUES_PER_BIT_PACKED_RUN * bit_width as usize) as i64,
- 8,
- );
- let max_rle_run_size =
- bit_util::MAX_VLQ_BYTE_LEN + bit_util::ceil(bit_width as i64, 8) as usize;
- std::cmp::max(max_bit_packed_run_size as usize, max_rle_run_size)
- }
-
- /// Returns the maximum buffer size takes to encode `num_values` values with
- /// `bit_width`.
- pub fn max_buffer_size(bit_width: u8, num_values: usize) -> usize {
- // First the maximum size for bit-packed run
- let bytes_per_run = bit_width;
- let num_runs = bit_util::ceil(num_values as i64, 8) as usize;
- let bit_packed_max_size = num_runs + num_runs * bytes_per_run as usize;
-
- // Second the maximum size for RLE run
- let min_rle_run_size = 1 + bit_util::ceil(bit_width as i64, 8) as usize;
- let rle_max_size =
- bit_util::ceil(num_values as i64, 8) as usize * min_rle_run_size;
- std::cmp::max(bit_packed_max_size, rle_max_size) as usize
- }
-
- /// Encodes `value`, which must be representable with `bit_width` bits.
- /// Returns true if the value fits in buffer, false if it doesn't, or
- /// error if something is wrong.
- #[inline]
- pub fn put(&mut self, value: u64) -> Result<bool> {
- // This function buffers 8 values at a time. After seeing 8 values, it
- // decides whether the current run should be encoded in bit-packed or RLE.
- if self.current_value == value {
- self.repeat_count += 1;
- if self.repeat_count > 8 {
- // A continuation of last value. No need to buffer.
- return Ok(true);
- }
- } else {
- if self.repeat_count >= 8 {
- // The current RLE run has ended and we've gathered enough. Flush first.
- assert_eq!(self.bit_packed_count, 0);
- self.flush_rle_run()?;
- }
- self.repeat_count = 1;
- self.current_value = value;
- }
-
- self.buffered_values[self.num_buffered_values] = value;
- self.num_buffered_values += 1;
- if self.num_buffered_values == 8 {
- // Buffered values are full. Flush them.
- assert_eq!(self.bit_packed_count % 8, 0);
- self.flush_buffered_values()?;
- }
-
- Ok(true)
- }
-
- #[inline]
- pub fn buffer(&self) -> &[u8] {
- self.bit_writer.buffer()
- }
-
- #[inline]
- pub fn len(&self) -> usize {
- self.bit_writer.bytes_written()
- }
-
- #[inline]
- pub fn consume(mut self) -> Result<Vec<u8>> {
- self.flush()?;
- Ok(self.bit_writer.consume())
- }
-
- /// Borrow equivalent of the `consume` method.
- /// Call `clear()` after invoking this method.
- #[inline]
- pub fn flush_buffer(&mut self) -> Result<&[u8]> {
- self.flush()?;
- Ok(self.bit_writer.flush_buffer())
- }
-
- /// Clears the internal state so this encoder can be reused (e.g., after becoming
- /// full).
- #[inline]
- pub fn clear(&mut self) {
- self.bit_writer.clear();
- self.num_buffered_values = 0;
- self.current_value = 0;
- self.repeat_count = 0;
- self.bit_packed_count = 0;
- self.indicator_byte_pos = -1;
- }
-
- /// Flushes all remaining values and return the final byte buffer maintained by the
- /// internal writer.
- #[inline]
- pub fn flush(&mut self) -> Result<()> {
- if self.bit_packed_count > 0
- || self.repeat_count > 0
- || self.num_buffered_values > 0
- {
- let all_repeat = self.bit_packed_count == 0
- && (self.repeat_count == self.num_buffered_values
- || self.num_buffered_values == 0);
- if self.repeat_count > 0 && all_repeat {
- self.flush_rle_run()?;
- } else {
- // Buffer the last group of bit-packed values to 8 by padding with 0s.
- if self.num_buffered_values > 0 {
- while self.num_buffered_values < 8 {
- self.buffered_values[self.num_buffered_values] = 0;
- self.num_buffered_values += 1;
- }
- }
- self.bit_packed_count += self.num_buffered_values;
- self.flush_bit_packed_run(true)?;
- self.repeat_count = 0;
- }
- }
- Ok(())
- }
-
- fn flush_rle_run(&mut self) -> Result<()> {
- assert!(self.repeat_count > 0);
- let indicator_value = self.repeat_count << 1;
- let mut result = self.bit_writer.put_vlq_int(indicator_value as u64);
- result &= self.bit_writer.put_aligned(
- self.current_value,
- bit_util::ceil(self.bit_width as i64, 8) as usize,
- );
- if !result {
- return Err(general_err!("Failed to write RLE run"));
- }
- self.num_buffered_values = 0;
- self.repeat_count = 0;
- Ok(())
- }
-
- fn flush_bit_packed_run(&mut self, update_indicator_byte: bool) -> Result<()> {
- if self.indicator_byte_pos < 0 {
- self.indicator_byte_pos = self.bit_writer.skip(1)? as i64;
- }
-
- // Write all buffered values as bit-packed literals
- for i in 0..self.num_buffered_values {
- let _ = self
- .bit_writer
- .put_value(self.buffered_values[i], self.bit_width as usize);
- }
- self.num_buffered_values = 0;
- if update_indicator_byte {
- // Write the indicator byte to the reserved position in `bit_writer`
- let num_groups = self.bit_packed_count / 8;
- let indicator_byte = ((num_groups << 1) | 1) as u8;
- if !self.bit_writer.put_aligned_offset(
- indicator_byte,
- 1,
- self.indicator_byte_pos as usize,
- ) {
- return Err(general_err!("Not enough space to write indicator byte"));
- }
- self.indicator_byte_pos = -1;
- self.bit_packed_count = 0;
- }
- Ok(())
- }
-
- #[inline(never)]
- fn flush_buffered_values(&mut self) -> Result<()> {
- if self.repeat_count >= 8 {
- self.num_buffered_values = 0;
- if self.bit_packed_count > 0 {
- // In this case we choose RLE encoding. Flush the current buffered values
- // as bit-packed encoding.
- assert_eq!(self.bit_packed_count % 8, 0);
- self.flush_bit_packed_run(true)?
- }
- return Ok(());
- }
-
- self.bit_packed_count += self.num_buffered_values;
- let num_groups = self.bit_packed_count / 8;
- if num_groups + 1 >= MAX_GROUPS_PER_BIT_PACKED_RUN {
- // We've reached the maximum value that can be hold in a single bit-packed
- // run.
- assert!(self.indicator_byte_pos >= 0);
- self.flush_bit_packed_run(true)?;
- } else {
- self.flush_bit_packed_run(false)?;
- }
- self.repeat_count = 0;
- Ok(())
- }
-}
-
-/// A RLE/Bit-Packing hybrid decoder.
-pub struct RleDecoder {
- // Number of bits used to encode the value. Must be between [0, 64].
- bit_width: u8,
-
- // Bit reader loaded with input buffer.
- bit_reader: Option<BitReader>,
-
- // Buffer used when `bit_reader` is not `None`, for batch reading.
- index_buf: [i32; 1024],
-
- // The remaining number of values in RLE for this run
- rle_left: u32,
-
- // The remaining number of values in Bit-Packing for this run
- bit_packed_left: u32,
-
- // The current value for the case of RLE mode
- current_value: Option<u64>,
-}
-
-impl RleDecoder {
- pub fn new(bit_width: u8) -> Self {
- RleDecoder {
- bit_width,
- rle_left: 0,
- bit_packed_left: 0,
- bit_reader: None,
- index_buf: [0; 1024],
- current_value: None,
- }
- }
-
- #[inline]
- pub fn set_data(&mut self, data: ByteBufferPtr) {
- if let Some(ref mut bit_reader) = self.bit_reader {
- bit_reader.reset(data);
- } else {
- self.bit_reader = Some(BitReader::new(data));
- }
-
- let _ = self.reload();
- }
-
- // These functions inline badly, they tend to inline and then create very large loop unrolls
- // that damage L1d-cache occupancy. This results in a ~18% performance drop
- #[inline(never)]
- pub fn get<T: FromBytes>(&mut self) -> Result<Option<T>> {
- assert!(size_of::<T>() <= 8);
-
- while self.rle_left == 0 && self.bit_packed_left == 0 {
- if !self.reload() {
- return Ok(None);
- }
- }
-
- let value = if self.rle_left > 0 {
- let rle_value = from_ne_slice(
- &self
- .current_value
- .as_mut()
- .expect("current_value should be Some")
- .to_ne_bytes(),
- );
- self.rle_left -= 1;
- rle_value
- } else {
- // self.bit_packed_left > 0
- let bit_reader = self.bit_reader.as_mut().expect("bit_reader should be Some");
- let bit_packed_value = bit_reader
- .get_value(self.bit_width as usize)
- .ok_or_else(|| eof_err!("Not enough data for 'bit_packed_value'"))?;
- self.bit_packed_left -= 1;
- bit_packed_value
- };
-
- Ok(Some(value))
- }
-
- #[inline(never)]
- pub fn get_batch<T: FromBytes>(&mut self, buffer: &mut [T]) -> Result<usize> {
- assert!(size_of::<T>() <= 8);
-
- let mut values_read = 0;
- while values_read < buffer.len() {
- if self.rle_left > 0 {
- let num_values =
- cmp::min(buffer.len() - values_read, self.rle_left as usize);
- for i in 0..num_values {
- let repeated_value = from_ne_slice(
- &self.current_value.as_mut().unwrap().to_ne_bytes(),
- );
- buffer[values_read + i] = repeated_value;
- }
- self.rle_left -= num_values as u32;
- values_read += num_values;
- } else if self.bit_packed_left > 0 {
- let mut num_values =
- cmp::min(buffer.len() - values_read, self.bit_packed_left as usize);
- let bit_reader =
- self.bit_reader.as_mut().expect("bit_reader should be set");
-
- num_values = bit_reader.get_batch::<T>(
- &mut buffer[values_read..values_read + num_values],
- self.bit_width as usize,
- );
- self.bit_packed_left -= num_values as u32;
- values_read += num_values;
- } else if !self.reload() {
- break;
- }
- }
-
- Ok(values_read)
- }
-
- #[inline(never)]
- pub fn get_batch_with_dict<T>(
- &mut self,
- dict: &[T],
- buffer: &mut [T],
- max_values: usize,
- ) -> Result<usize>
- where
- T: Default + Clone,
- {
- assert!(buffer.len() >= max_values);
-
- let mut values_read = 0;
- while values_read < max_values {
- if self.rle_left > 0 {
- let num_values =
- cmp::min(max_values - values_read, self.rle_left as usize);
- let dict_idx = self.current_value.unwrap() as usize;
- for i in 0..num_values {
- buffer[values_read + i].clone_from(&dict[dict_idx]);
- }
- self.rle_left -= num_values as u32;
- values_read += num_values;
- } else if self.bit_packed_left > 0 {
- let bit_reader =
- self.bit_reader.as_mut().expect("bit_reader should be set");
-
- let mut num_values =
- cmp::min(max_values - values_read, self.bit_packed_left as usize);
-
- num_values = cmp::min(num_values, self.index_buf.len());
- loop {
- num_values = bit_reader.get_batch::<i32>(
- &mut self.index_buf[..num_values],
- self.bit_width as usize,
- );
- for i in 0..num_values {
- buffer[values_read + i]
- .clone_from(&dict[self.index_buf[i] as usize])
- }
- self.bit_packed_left -= num_values as u32;
- values_read += num_values;
- if num_values < self.index_buf.len() {
- break;
- }
- }
- } else if !self.reload() {
- break;
- }
- }
-
- Ok(values_read)
- }
-
- #[inline]
- fn reload(&mut self) -> bool {
- let bit_reader = self.bit_reader.as_mut().expect("bit_reader should be set");
-
- if let Some(indicator_value) = bit_reader.get_vlq_int() {
- if indicator_value & 1 == 1 {
- self.bit_packed_left = ((indicator_value >> 1) * 8) as u32;
- } else {
- self.rle_left = (indicator_value >> 1) as u32;
- let value_width = bit_util::ceil(self.bit_width as i64, 8);
- self.current_value = bit_reader.get_aligned::<u64>(value_width as usize);
- assert!(self.current_value.is_some());
- }
- true
- } else {
- false
- }
- }
-}
-
-#[cfg(test)]
-mod tests {
- use super::*;
-
- use rand::{self, distributions::Standard, thread_rng, Rng, SeedableRng};
-
- use crate::util::memory::ByteBufferPtr;
-
- const MAX_WIDTH: usize = 32;
-
- #[test]
- fn test_rle_decode_int32() {
- // Test data: 0-7 with bit width 3
- // 00000011 10001000 11000110 11111010
- let data = ByteBufferPtr::new(vec![0x03, 0x88, 0xC6, 0xFA]);
- let mut decoder: RleDecoder = RleDecoder::new(3);
- decoder.set_data(data);
- let mut buffer = vec![0; 8];
- let expected = vec![0, 1, 2, 3, 4, 5, 6, 7];
- let result = decoder.get_batch::<i32>(&mut buffer);
- assert!(result.is_ok());
- assert_eq!(buffer, expected);
- }
-
- #[test]
- fn test_rle_consume_flush_buffer() {
- let data = vec![1, 1, 1, 2, 2, 3, 3, 3];
- let mut encoder1 = RleEncoder::new(3, 256);
- let mut encoder2 = RleEncoder::new(3, 256);
- for value in data {
- encoder1.put(value as u64).unwrap();
- encoder2.put(value as u64).unwrap();
- }
- let res1 = encoder1.flush_buffer().unwrap();
- let res2 = encoder2.consume().unwrap();
- assert_eq!(res1, &res2[..]);
- }
-
- #[test]
- fn test_rle_decode_bool() {
- // RLE test data: 50 1s followed by 50 0s
- // 01100100 00000001 01100100 00000000
- let data1 = ByteBufferPtr::new(vec![0x64, 0x01, 0x64, 0x00]);
-
- // Bit-packing test data: alternating 1s and 0s, 100 total
- // 100 / 8 = 13 groups
- // 00011011 10101010 ... 00001010
- let data2 = ByteBufferPtr::new(vec![
- 0x1B, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA,
- 0x0A,
- ]);
-
- let mut decoder: RleDecoder = RleDecoder::new(1);
- decoder.set_data(data1);
- let mut buffer = vec![false; 100];
- let mut expected = vec![];
- for i in 0..100 {
- if i < 50 {
- expected.push(true);
- } else {
- expected.push(false);
- }
- }
- let result = decoder.get_batch::<bool>(&mut buffer);
- assert!(result.is_ok());
- assert_eq!(buffer, expected);
-
- decoder.set_data(data2);
- let mut buffer = vec![false; 100];
- let mut expected = vec![];
- for i in 0..100 {
- if i % 2 == 0 {
- expected.push(false);
- } else {
- expected.push(true);
- }
- }
- let result = decoder.get_batch::<bool>(&mut buffer);
- assert!(result.is_ok());
- assert_eq!(buffer, expected);
- }
-
- #[test]
- fn test_rle_decode_with_dict_int32() {
- // Test RLE encoding: 3 0s followed by 4 1s followed by 5 2s
- // 00000110 00000000 00001000 00000001 00001010 00000010
- let dict = vec![10, 20, 30];
- let data = ByteBufferPtr::new(vec![0x06, 0x00, 0x08, 0x01, 0x0A, 0x02]);
- let mut decoder: RleDecoder = RleDecoder::new(3);
- decoder.set_data(data);
- let mut buffer = vec![0; 12];
- let expected = vec![10, 10, 10, 20, 20, 20, 20, 30, 30, 30, 30, 30];
- let result = decoder.get_batch_with_dict::<i32>(&dict, &mut buffer, 12);
- assert!(result.is_ok());
- assert_eq!(buffer, expected);
-
- // Test bit-pack encoding: 345345345455 (2 groups: 8 and 4)
- // 011 100 101 011 100 101 011 100 101 100 101 101
- // 00000011 01100011 11000111 10001110 00000011 01100101 00001011
- let dict = vec!["aaa", "bbb", "ccc", "ddd", "eee", "fff"];
- let data = ByteBufferPtr::new(vec![0x03, 0x63, 0xC7, 0x8E, 0x03, 0x65, 0x0B]);
- let mut decoder: RleDecoder = RleDecoder::new(3);
- decoder.set_data(data);
- let mut buffer = vec![""; 12];
- let expected = vec![
- "ddd", "eee", "fff", "ddd", "eee", "fff", "ddd", "eee", "fff", "eee", "fff",
- "fff",
- ];
- let result = decoder.get_batch_with_dict::<&str>(
- dict.as_slice(),
- buffer.as_mut_slice(),
- 12,
- );
- assert!(result.is_ok());
- assert_eq!(buffer, expected);
- }
-
- fn validate_rle(
- values: &[i64],
- bit_width: u8,
- expected_encoding: Option<&[u8]>,
- expected_len: i32,
- ) {
- let buffer_len = 64 * 1024;
- let mut encoder = RleEncoder::new(bit_width, buffer_len);
- for v in values {
- let result = encoder.put(*v as u64);
- assert!(result.is_ok());
- }
- let buffer = ByteBufferPtr::new(encoder.consume().expect("Expect consume() OK"));
- if expected_len != -1 {
- assert_eq!(buffer.len(), expected_len as usize);
- }
- if let Some(b) = expected_encoding {
- assert_eq!(buffer.as_ref(), b);
- }
-
- // Verify read
- let mut decoder = RleDecoder::new(bit_width);
- decoder.set_data(buffer.all());
- for v in values {
- let val: i64 = decoder
- .get()
- .expect("get() should be OK")
- .expect("get() should return more value");
- assert_eq!(val, *v);
- }
-
- // Verify batch read
- decoder.set_data(buffer);
- let mut values_read: Vec<i64> = vec![0; values.len()];
- decoder
- .get_batch(&mut values_read[..])
- .expect("get_batch() should be OK");
- assert_eq!(&values_read[..], values);
- }
-
- #[test]
- fn test_rle_specific_sequences() {
- let mut expected_buffer = Vec::new();
- let mut values = Vec::new();
- for _ in 0..50 {
- values.push(0);
- }
- for _ in 0..50 {
- values.push(1);
- }
- expected_buffer.push(50 << 1);
- expected_buffer.push(0);
- expected_buffer.push(50 << 1);
- expected_buffer.push(1);
-
- for width in 1..9 {
- validate_rle(&values[..], width, Some(&expected_buffer[..]), 4);
- }
- for width in 9..MAX_WIDTH + 1 {
- validate_rle(
- &values[..],
- width as u8,
- None,
- 2 * (1 + bit_util::ceil(width as i64, 8) as i32),
- );
- }
-
- // Test 100 0's and 1's alternating
- values.clear();
- expected_buffer.clear();
- for i in 0..101 {
- values.push(i % 2);
- }
- let num_groups = bit_util::ceil(100, 8) as u8;
- expected_buffer.push(((num_groups << 1) as u8) | 1);
- for _ in 1..(100 / 8) + 1 {
- expected_buffer.push(0b10101010);
- }
- // For the last 4 0 and 1's, padded with 0.
- expected_buffer.push(0b00001010);
- validate_rle(
- &values,
- 1,
- Some(&expected_buffer[..]),
- 1 + num_groups as i32,
- );
- for width in 2..MAX_WIDTH + 1 {
- let num_values = bit_util::ceil(100, 8) * 8;
- validate_rle(
- &values,
- width as u8,
- None,
- 1 + bit_util::ceil(width as i64 * num_values, 8) as i32,
- );
- }
- }
-
- // `validate_rle` on `num_vals` with width `bit_width`. If `value` is -1, that value
- // is used, otherwise alternating values are used.
- fn test_rle_values(bit_width: usize, num_vals: usize, value: i32) {
- let mod_val = if bit_width == 64 {
- 1
- } else {
- 1u64 << bit_width
- };
- let mut values: Vec<i64> = vec![];
- for v in 0..num_vals {
- let val = if value == -1 {
- v as i64 % mod_val as i64
- } else {
- value as i64
- };
- values.push(val);
- }
- validate_rle(&values, bit_width as u8, None, -1);
- }
-
- #[test]
- fn test_values() {
- for width in 1..MAX_WIDTH + 1 {
- test_rle_values(width, 1, -1);
- test_rle_values(width, 1024, -1);
- test_rle_values(width, 1024, 0);
- test_rle_values(width, 1024, 1);
- }
- }
-
- #[test]
- fn test_rle_specific_roundtrip() {
- let bit_width = 1;
- let buffer_len = RleEncoder::min_buffer_size(bit_width);
- let values: Vec<i16> = vec![0, 1, 1, 1, 1, 0, 0, 0, 0, 1];
- let mut encoder = RleEncoder::new(bit_width, buffer_len);
- for v in &values {
- assert!(encoder.put(*v as u64).expect("put() should be OK"));
- }
- let buffer = encoder.consume().expect("consume() should be OK");
- let mut decoder = RleDecoder::new(bit_width);
- decoder.set_data(ByteBufferPtr::new(buffer));
- let mut actual_values: Vec<i16> = vec![0; values.len()];
- decoder
- .get_batch(&mut actual_values)
- .expect("get_batch() should be OK");
- assert_eq!(actual_values, values);
- }
-
- fn test_round_trip(values: &[i32], bit_width: u8) {
- let buffer_len = 64 * 1024;
- let mut encoder = RleEncoder::new(bit_width, buffer_len);
- for v in values {
- let result = encoder.put(*v as u64).expect("put() should be OK");
- assert!(result, "put() should not return false");
- }
-
- let buffer =
- ByteBufferPtr::new(encoder.consume().expect("consume() should be OK"));
-
- // Verify read
- let mut decoder = RleDecoder::new(bit_width);
- decoder.set_data(buffer.all());
- for v in values {
- let val = decoder
- .get::<i32>()
- .expect("get() should be OK")
- .expect("get() should return value");
- assert_eq!(val, *v);
- }
-
- // Verify batch read
- let mut decoder = RleDecoder::new(bit_width);
- decoder.set_data(buffer);
- let mut values_read: Vec<i32> = vec![0; values.len()];
- decoder
- .get_batch(&mut values_read[..])
- .expect("get_batch() should be OK");
- assert_eq!(&values_read[..], values);
- }
-
- #[test]
- fn test_random() {
- let seed_len = 32;
- let niters = 50;
- let ngroups = 1000;
- let max_group_size = 15;
- let mut values = vec![];
-
- for _ in 0..niters {
- values.clear();
- let rng = thread_rng();
- let seed_vec: Vec<u8> =
- rng.sample_iter::<u8, _>(&Standard).take(seed_len).collect();
- let mut seed = [0u8; 32];
- seed.copy_from_slice(&seed_vec[0..seed_len]);
- let mut gen = rand::rngs::StdRng::from_seed(seed);
-
- let mut parity = false;
- for _ in 0..ngroups {
- let mut group_size = gen.gen_range(1..20);
- if group_size > max_group_size {
- group_size = 1;
- }
- for _ in 0..group_size {
- values.push(parity as i32);
- }
- parity = !parity;
- }
- let bit_width = bit_util::num_required_bits(values.len() as u64);
- assert!(bit_width < 64);
- test_round_trip(&values[..], bit_width as u8);
- }
- }
-}
diff --git a/parquet/src/errors.rs b/parquet/src/errors.rs
deleted file mode 100644
index be1a221..0000000
--- a/parquet/src/errors.rs
+++ /dev/null
@@ -1,146 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-//! Common Parquet errors and macros.
-
-use std::{cell, convert, io, result, str};
-
-#[cfg(any(feature = "arrow", test))]
-use arrow::error::ArrowError;
-
-#[derive(Debug, PartialEq, Clone)]
-pub enum ParquetError {
- /// General Parquet error.
- /// Returned when code violates normal workflow of working with Parquet files.
- General(String),
- /// "Not yet implemented" Parquet error.
- /// Returned when functionality is not yet available.
- NYI(String),
- /// "End of file" Parquet error.
- /// Returned when IO related failures occur, e.g. when there are not enough bytes to
- /// decode.
- EOF(String),
- #[cfg(any(feature = "arrow", test))]
- /// Arrow error.
- /// Returned when reading into arrow or writing from arrow.
- ArrowError(String),
- IndexOutOfBound(usize, usize),
-}
-
-impl std::fmt::Display for ParquetError {
- fn fmt(&self, fmt: &mut std::fmt::Formatter) -> std::fmt::Result {
- match *self {
- ParquetError::General(ref message) => {
- write!(fmt, "Parquet error: {}", message)
- }
- ParquetError::NYI(ref message) => write!(fmt, "NYI: {}", message),
- ParquetError::EOF(ref message) => write!(fmt, "EOF: {}", message),
- #[cfg(any(feature = "arrow", test))]
- ParquetError::ArrowError(ref message) => write!(fmt, "Arrow: {}", message),
- ParquetError::IndexOutOfBound(ref index, ref bound) => {
- write!(fmt, "Index {} out of bound: {}", index, bound)
- }
- }
- }
-}
-
-impl std::error::Error for ParquetError {
- fn cause(&self) -> Option<&dyn ::std::error::Error> {
- None
- }
-}
-
-impl From<io::Error> for ParquetError {
- fn from(e: io::Error) -> ParquetError {
- ParquetError::General(format!("underlying IO error: {}", e))
- }
-}
-
-#[cfg(any(feature = "snap", test))]
-impl From<snap::Error> for ParquetError {
- fn from(e: snap::Error) -> ParquetError {
- ParquetError::General(format!("underlying snap error: {}", e))
- }
-}
-
-impl From<thrift::Error> for ParquetError {
- fn from(e: thrift::Error) -> ParquetError {
- ParquetError::General(format!("underlying Thrift error: {}", e))
- }
-}
-
-impl From<cell::BorrowMutError> for ParquetError {
- fn from(e: cell::BorrowMutError) -> ParquetError {
- ParquetError::General(format!("underlying borrow error: {}", e))
- }
-}
-
-impl From<str::Utf8Error> for ParquetError {
- fn from(e: str::Utf8Error) -> ParquetError {
- ParquetError::General(format!("underlying utf8 error: {}", e))
- }
-}
-
-#[cfg(any(feature = "arrow", test))]
-impl From<ArrowError> for ParquetError {
- fn from(e: ArrowError) -> ParquetError {
- ParquetError::ArrowError(format!("underlying Arrow error: {}", e))
- }
-}
-
-/// A specialized `Result` for Parquet errors.
-pub type Result<T> = result::Result<T, ParquetError>;
-
-// ----------------------------------------------------------------------
-// Conversion from `ParquetError` to other types of `Error`s
-
-impl convert::From<ParquetError> for io::Error {
- fn from(e: ParquetError) -> Self {
- io::Error::new(io::ErrorKind::Other, e)
- }
-}
-
-// ----------------------------------------------------------------------
-// Convenient macros for different errors
-
-macro_rules! general_err {
- ($fmt:expr) => (ParquetError::General($fmt.to_owned()));
- ($fmt:expr, $($args:expr),*) => (ParquetError::General(format!($fmt, $($args),*)));
- ($e:expr, $fmt:expr) => (ParquetError::General($fmt.to_owned(), $e));
- ($e:ident, $fmt:expr, $($args:tt),*) => (
- ParquetError::General(&format!($fmt, $($args),*), $e));
-}
-
-macro_rules! nyi_err {
- ($fmt:expr) => (ParquetError::NYI($fmt.to_owned()));
- ($fmt:expr, $($args:expr),*) => (ParquetError::NYI(format!($fmt, $($args),*)));
-}
-
-macro_rules! eof_err {
- ($fmt:expr) => (ParquetError::EOF($fmt.to_owned()));
- ($fmt:expr, $($args:expr),*) => (ParquetError::EOF(format!($fmt, $($args),*)));
-}
-
-// ----------------------------------------------------------------------
-// Convert parquet error into other errors
-
-#[cfg(any(feature = "arrow", test))]
-impl Into<ArrowError> for ParquetError {
- fn into(self) -> ArrowError {
- ArrowError::ParquetError(format!("{}", self))
- }
-}
diff --git a/parquet/src/file/footer.rs b/parquet/src/file/footer.rs
deleted file mode 100644
index 2e57294..0000000
--- a/parquet/src/file/footer.rs
+++ /dev/null
@@ -1,263 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-use std::{
- cmp::min,
- io::{Cursor, Read, Seek, SeekFrom},
- sync::Arc,
-};
-
-use byteorder::{ByteOrder, LittleEndian};
-use parquet_format::{ColumnOrder as TColumnOrder, FileMetaData as TFileMetaData};
-use thrift::protocol::TCompactInputProtocol;
-
-use crate::basic::ColumnOrder;
-
-use crate::errors::{ParquetError, Result};
-use crate::file::{
- metadata::*, reader::ChunkReader, DEFAULT_FOOTER_READ_SIZE, FOOTER_SIZE,
- PARQUET_MAGIC,
-};
-
-use crate::schema::types::{self, SchemaDescriptor};
-
-/// Layout of Parquet file
-/// +---------------------------+-----+---+
-/// | Rest of file | B | A |
-/// +---------------------------+-----+---+
-/// where A: parquet footer, B: parquet metadata.
-///
-/// The reader first reads DEFAULT_FOOTER_SIZE bytes from the end of the file.
-/// If it is not enough according to the length indicated in the footer, it reads more bytes.
-pub fn parse_metadata<R: ChunkReader>(chunk_reader: &R) -> Result<ParquetMetaData> {
- // check file is large enough to hold footer
- let file_size = chunk_reader.len();
- if file_size < (FOOTER_SIZE as u64) {
- return Err(general_err!(
- "Invalid Parquet file. Size is smaller than footer"
- ));
- }
-
- // read and cache up to DEFAULT_FOOTER_READ_SIZE bytes from the end and process the footer
- let default_end_len = min(DEFAULT_FOOTER_READ_SIZE, chunk_reader.len() as usize);
- let mut default_end_reader = chunk_reader
- .get_read(chunk_reader.len() - default_end_len as u64, default_end_len)?;
- let mut default_len_end_buf = vec![0; default_end_len];
- default_end_reader.read_exact(&mut default_len_end_buf)?;
-
- // check this is indeed a parquet file
- if default_len_end_buf[default_end_len - 4..] != PARQUET_MAGIC {
- return Err(general_err!("Invalid Parquet file. Corrupt footer"));
- }
-
- // get the metadata length from the footer
- let metadata_len = LittleEndian::read_i32(
- &default_len_end_buf[default_end_len - 8..default_end_len - 4],
- ) as i64;
- if metadata_len < 0 {
- return Err(general_err!(
- "Invalid Parquet file. Metadata length is less than zero ({})",
- metadata_len
- ));
- }
- let footer_metadata_len = FOOTER_SIZE + metadata_len as usize;
-
- // build up the reader covering the entire metadata
- let mut default_end_cursor = Cursor::new(default_len_end_buf);
- let metadata_read: Box<dyn Read>;
- if footer_metadata_len > file_size as usize {
- return Err(general_err!(
- "Invalid Parquet file. Metadata start is less than zero ({})",
- file_size as i64 - footer_metadata_len as i64
- ));
- } else if footer_metadata_len < DEFAULT_FOOTER_READ_SIZE {
- // the whole metadata is in the bytes we already read
- default_end_cursor.seek(SeekFrom::End(-(footer_metadata_len as i64)))?;
- metadata_read = Box::new(default_end_cursor);
- } else {
- // the end of file read by default is not long enough, read missing bytes
- let complementary_end_read = chunk_reader.get_read(
- file_size - footer_metadata_len as u64,
- FOOTER_SIZE + metadata_len as usize - default_end_len,
- )?;
- metadata_read = Box::new(complementary_end_read.chain(default_end_cursor));
- }
-
- // TODO: row group filtering
- let mut prot = TCompactInputProtocol::new(metadata_read);
- let t_file_metadata: TFileMetaData = TFileMetaData::read_from_in_protocol(&mut prot)
- .map_err(|e| ParquetError::General(format!("Could not parse metadata: {}", e)))?;
- let schema = types::from_thrift(&t_file_metadata.schema)?;
- let schema_descr = Arc::new(SchemaDescriptor::new(schema));
- let mut row_groups = Vec::new();
- for rg in t_file_metadata.row_groups {
- row_groups.push(RowGroupMetaData::from_thrift(schema_descr.clone(), rg)?);
- }
- let column_orders = parse_column_orders(t_file_metadata.column_orders, &schema_descr);
-
- let file_metadata = FileMetaData::new(
- t_file_metadata.version,
- t_file_metadata.num_rows,
- t_file_metadata.created_by,
- t_file_metadata.key_value_metadata,
- schema_descr,
- column_orders,
- );
- Ok(ParquetMetaData::new(file_metadata, row_groups))
-}
-
-/// Parses column orders from Thrift definition.
-/// If no column orders are defined, returns `None`.
-fn parse_column_orders(
- t_column_orders: Option<Vec<TColumnOrder>>,
- schema_descr: &SchemaDescriptor,
-) -> Option<Vec<ColumnOrder>> {
- match t_column_orders {
- Some(orders) => {
- // Should always be the case
- assert_eq!(
- orders.len(),
- schema_descr.num_columns(),
- "Column order length mismatch"
- );
- let mut res = Vec::new();
- for (i, column) in schema_descr.columns().iter().enumerate() {
- match orders[i] {
- TColumnOrder::TYPEORDER(_) => {
- let sort_order = ColumnOrder::get_sort_order(
- column.logical_type(),
- column.converted_type(),
- column.physical_type(),
- );
- res.push(ColumnOrder::TYPE_DEFINED_ORDER(sort_order));
- }
- }
- }
- Some(res)
- }
- None => None,
- }
-}
-
-#[cfg(test)]
-mod tests {
- use super::*;
-
- use crate::basic::SortOrder;
- use crate::basic::Type;
- use crate::schema::types::Type as SchemaType;
- use crate::util::test_common::get_temp_file;
- use parquet_format::TypeDefinedOrder;
-
- #[test]
- fn test_parse_metadata_size_smaller_than_footer() {
- let test_file = get_temp_file("corrupt-1.parquet", &[]);
- let reader_result = parse_metadata(&test_file);
- assert!(reader_result.is_err());
- assert_eq!(
- reader_result.err().unwrap(),
- general_err!("Invalid Parquet file. Size is smaller than footer")
- );
- }
-
- #[test]
- fn test_parse_metadata_corrupt_footer() {
- let test_file = get_temp_file("corrupt-2.parquet", &[1, 2, 3, 4, 5, 6, 7, 8]);
- let reader_result = parse_metadata(&test_file);
- assert!(reader_result.is_err());
- assert_eq!(
- reader_result.err().unwrap(),
- general_err!("Invalid Parquet file. Corrupt footer")
- );
- }
-
- #[test]
- fn test_parse_metadata_invalid_length() {
- let test_file =
- get_temp_file("corrupt-3.parquet", &[0, 0, 0, 255, b'P', b'A', b'R', b'1']);
- let reader_result = parse_metadata(&test_file);
- assert!(reader_result.is_err());
- assert_eq!(
- reader_result.err().unwrap(),
- general_err!(
- "Invalid Parquet file. Metadata length is less than zero (-16777216)"
- )
- );
- }
-
- #[test]
- fn test_parse_metadata_invalid_start() {
- let test_file =
- get_temp_file("corrupt-4.parquet", &[255, 0, 0, 0, b'P', b'A', b'R', b'1']);
- let reader_result = parse_metadata(&test_file);
- assert!(reader_result.is_err());
- assert_eq!(
- reader_result.err().unwrap(),
- general_err!("Invalid Parquet file. Metadata start is less than zero (-255)")
- );
- }
-
- #[test]
- fn test_metadata_column_orders_parse() {
- // Define simple schema, we do not need to provide logical types.
- let mut fields = vec![
- Arc::new(
- SchemaType::primitive_type_builder("col1", Type::INT32)
- .build()
- .unwrap(),
- ),
- Arc::new(
- SchemaType::primitive_type_builder("col2", Type::FLOAT)
- .build()
- .unwrap(),
- ),
- ];
- let schema = SchemaType::group_type_builder("schema")
- .with_fields(&mut fields)
- .build()
- .unwrap();
- let schema_descr = SchemaDescriptor::new(Arc::new(schema));
-
- let t_column_orders = Some(vec![
- TColumnOrder::TYPEORDER(TypeDefinedOrder::new()),
- TColumnOrder::TYPEORDER(TypeDefinedOrder::new()),
- ]);
-
- assert_eq!(
- parse_column_orders(t_column_orders, &schema_descr),
- Some(vec![
- ColumnOrder::TYPE_DEFINED_ORDER(SortOrder::SIGNED),
- ColumnOrder::TYPE_DEFINED_ORDER(SortOrder::SIGNED)
- ])
- );
-
- // Test when no column orders are defined.
- assert_eq!(parse_column_orders(None, &schema_descr), None);
- }
-
- #[test]
- #[should_panic(expected = "Column order length mismatch")]
- fn test_metadata_column_orders_len_mismatch() {
- let schema = SchemaType::group_type_builder("schema").build().unwrap();
- let schema_descr = SchemaDescriptor::new(Arc::new(schema));
-
- let t_column_orders =
- Some(vec![TColumnOrder::TYPEORDER(TypeDefinedOrder::new())]);
-
- parse_column_orders(t_column_orders, &schema_descr);
- }
-}
diff --git a/parquet/src/file/metadata.rs b/parquet/src/file/metadata.rs
deleted file mode 100644
index 150c42c..0000000
--- a/parquet/src/file/metadata.rs
+++ /dev/null
@@ -1,789 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-//! Contains information about available Parquet metadata.
-//!
-//! The hierarchy of metadata is as follows:
-//!
-//! [`ParquetMetaData`](struct.ParquetMetaData.html) contains
-//! [`FileMetaData`](struct.FileMetaData.html) and zero or more
-//! [`RowGroupMetaData`](struct.RowGroupMetaData.html) for each row group.
-//!
-//! [`FileMetaData`](struct.FileMetaData.html) includes file version, application specific
-//! metadata.
-//!
-//! Each [`RowGroupMetaData`](struct.RowGroupMetaData.html) contains information about row
-//! group and one or more [`ColumnChunkMetaData`](struct.ColumnChunkMetaData.html) for
-//! each column chunk.
-//!
-//! [`ColumnChunkMetaData`](struct.ColumnChunkMetaData.html) has information about column
-//! chunk (primitive leaf column), including encoding/compression, number of values, etc.
-
-use std::sync::Arc;
-
-use parquet_format::{ColumnChunk, ColumnMetaData, RowGroup};
-
-use crate::basic::{ColumnOrder, Compression, Encoding, Type};
-use crate::errors::{ParquetError, Result};
-use crate::file::statistics::{self, Statistics};
-use crate::schema::types::{
- ColumnDescPtr, ColumnDescriptor, ColumnPath, SchemaDescPtr, SchemaDescriptor,
- Type as SchemaType,
-};
-
-/// Global Parquet metadata.
-#[derive(Debug, Clone)]
-pub struct ParquetMetaData {
- file_metadata: FileMetaData,
- row_groups: Vec<RowGroupMetaData>,
-}
-
-impl ParquetMetaData {
- /// Creates Parquet metadata from file metadata and a list of row group metadata `Arc`s
- /// for each available row group.
- pub fn new(file_metadata: FileMetaData, row_groups: Vec<RowGroupMetaData>) -> Self {
- ParquetMetaData {
- file_metadata,
- row_groups,
- }
- }
-
- /// Returns file metadata as reference.
- pub fn file_metadata(&self) -> &FileMetaData {
- &self.file_metadata
- }
-
- /// Returns number of row groups in this file.
- pub fn num_row_groups(&self) -> usize {
- self.row_groups.len()
- }
-
- /// Returns row group metadata for `i`th position.
- /// Position should be less than number of row groups `num_row_groups`.
- pub fn row_group(&self, i: usize) -> &RowGroupMetaData {
- &self.row_groups[i]
- }
-
- /// Returns slice of row groups in this file.
- pub fn row_groups(&self) -> &[RowGroupMetaData] {
- &self.row_groups
- }
-}
-
-pub type KeyValue = parquet_format::KeyValue;
-
-/// Reference counted pointer for [`FileMetaData`].
-pub type FileMetaDataPtr = Arc<FileMetaData>;
-
-/// Metadata for a Parquet file.
-#[derive(Debug, Clone)]
-pub struct FileMetaData {
- version: i32,
- num_rows: i64,
- created_by: Option<String>,
- key_value_metadata: Option<Vec<KeyValue>>,
- schema_descr: SchemaDescPtr,
- column_orders: Option<Vec<ColumnOrder>>,
-}
-
-impl FileMetaData {
- /// Creates new file metadata.
- pub fn new(
- version: i32,
- num_rows: i64,
- created_by: Option<String>,
- key_value_metadata: Option<Vec<KeyValue>>,
- schema_descr: SchemaDescPtr,
- column_orders: Option<Vec<ColumnOrder>>,
- ) -> Self {
- FileMetaData {
- version,
- num_rows,
- created_by,
- key_value_metadata,
- schema_descr,
- column_orders,
- }
- }
-
- /// Returns version of this file.
- pub fn version(&self) -> i32 {
- self.version
- }
-
- /// Returns number of rows in the file.
- pub fn num_rows(&self) -> i64 {
- self.num_rows
- }
-
- /// String message for application that wrote this file.
- ///
- /// This should have the following format:
- /// `<application> version <application version> (build <application build hash>)`.
- ///
- /// ```shell
- /// parquet-mr version 1.8.0 (build 0fda28af84b9746396014ad6a415b90592a98b3b)
- /// ```
- pub fn created_by(&self) -> &Option<String> {
- &self.created_by
- }
-
- /// Returns key_value_metadata of this file.
- pub fn key_value_metadata(&self) -> &Option<Vec<KeyValue>> {
- &self.key_value_metadata
- }
-
- /// Returns Parquet ['Type`] that describes schema in this file.
- pub fn schema(&self) -> &SchemaType {
- self.schema_descr.root_schema()
- }
-
- /// Returns a reference to schema descriptor.
- pub fn schema_descr(&self) -> &SchemaDescriptor {
- &self.schema_descr
- }
-
- /// Returns reference counted clone for schema descriptor.
- pub fn schema_descr_ptr(&self) -> SchemaDescPtr {
- self.schema_descr.clone()
- }
-
- /// Column (sort) order used for `min` and `max` values of each column in this file.
- ///
- /// Each column order corresponds to one column, determined by its position in the
- /// list, matching the position of the column in the schema.
- ///
- /// When `None` is returned, there are no column orders available, and each column
- /// should be assumed to have undefined (legacy) column order.
- pub fn column_orders(&self) -> Option<&Vec<ColumnOrder>> {
- self.column_orders.as_ref()
- }
-
- /// Returns column order for `i`th column in this file.
- /// If column orders are not available, returns undefined (legacy) column order.
- pub fn column_order(&self, i: usize) -> ColumnOrder {
- self.column_orders
- .as_ref()
- .map(|data| data[i])
- .unwrap_or(ColumnOrder::UNDEFINED)
- }
-}
-
-/// Reference counted pointer for [`RowGroupMetaData`].
-pub type RowGroupMetaDataPtr = Arc<RowGroupMetaData>;
-
-/// Metadata for a row group.
-#[derive(Debug, Clone)]
-pub struct RowGroupMetaData {
- columns: Vec<ColumnChunkMetaData>,
- num_rows: i64,
- total_byte_size: i64,
- schema_descr: SchemaDescPtr,
-}
-
-impl RowGroupMetaData {
- /// Returns builer for row group metadata.
- pub fn builder(schema_descr: SchemaDescPtr) -> RowGroupMetaDataBuilder {
- RowGroupMetaDataBuilder::new(schema_descr)
- }
-
- /// Number of columns in this row group.
- pub fn num_columns(&self) -> usize {
- self.columns.len()
- }
-
- /// Returns column chunk metadata for `i`th column.
- pub fn column(&self, i: usize) -> &ColumnChunkMetaData {
- &self.columns[i]
- }
-
- /// Returns slice of column chunk metadata.
- pub fn columns(&self) -> &[ColumnChunkMetaData] {
- &self.columns
- }
-
- /// Number of rows in this row group.
- pub fn num_rows(&self) -> i64 {
- self.num_rows
- }
-
- /// Total byte size of all uncompressed column data in this row group.
- pub fn total_byte_size(&self) -> i64 {
- self.total_byte_size
- }
-
- /// Total size of all compressed column data in this row group.
- pub fn compressed_size(&self) -> i64 {
- self.columns.iter().map(|c| c.total_compressed_size).sum()
- }
-
- /// Returns reference to a schema descriptor.
- pub fn schema_descr(&self) -> &SchemaDescriptor {
- self.schema_descr.as_ref()
- }
-
- /// Returns reference counted clone of schema descriptor.
- pub fn schema_descr_ptr(&self) -> SchemaDescPtr {
- self.schema_descr.clone()
- }
-
- /// Method to convert from Thrift.
- pub fn from_thrift(
- schema_descr: SchemaDescPtr,
- mut rg: RowGroup,
- ) -> Result<RowGroupMetaData> {
- assert_eq!(schema_descr.num_columns(), rg.columns.len());
- let total_byte_size = rg.total_byte_size;
- let num_rows = rg.num_rows;
- let mut columns = vec![];
- for (c, d) in rg.columns.drain(0..).zip(schema_descr.columns()) {
- let cc = ColumnChunkMetaData::from_thrift(d.clone(), c)?;
- columns.push(cc);
- }
- Ok(RowGroupMetaData {
- columns,
- num_rows,
- total_byte_size,
- schema_descr,
- })
- }
-
- /// Method to convert to Thrift.
- pub fn to_thrift(&self) -> RowGroup {
- RowGroup {
- columns: self.columns().iter().map(|v| v.to_thrift()).collect(),
- total_byte_size: self.total_byte_size,
- num_rows: self.num_rows,
- sorting_columns: None,
- }
- }
-}
-
-/// Builder for row group metadata.
-pub struct RowGroupMetaDataBuilder {
- columns: Vec<ColumnChunkMetaData>,
- schema_descr: SchemaDescPtr,
- num_rows: i64,
- total_byte_size: i64,
-}
-
-impl RowGroupMetaDataBuilder {
- /// Creates new builder from schema descriptor.
- fn new(schema_descr: SchemaDescPtr) -> Self {
- Self {
- columns: Vec::with_capacity(schema_descr.num_columns()),
- schema_descr,
- num_rows: 0,
- total_byte_size: 0,
- }
- }
-
- /// Sets number of rows in this row group.
- pub fn set_num_rows(mut self, value: i64) -> Self {
- self.num_rows = value;
- self
- }
-
- /// Sets total size in bytes for this row group.
- pub fn set_total_byte_size(mut self, value: i64) -> Self {
- self.total_byte_size = value;
- self
- }
-
- /// Sets column metadata for this row group.
- pub fn set_column_metadata(mut self, value: Vec<ColumnChunkMetaData>) -> Self {
- self.columns = value;
- self
- }
-
- /// Builds row group metadata.
- pub fn build(self) -> Result<RowGroupMetaData> {
- if self.schema_descr.num_columns() != self.columns.len() {
- return Err(general_err!(
- "Column length mismatch: {} != {}",
- self.schema_descr.num_columns(),
- self.columns.len()
- ));
- }
-
- Ok(RowGroupMetaData {
- columns: self.columns,
- num_rows: self.num_rows,
- total_byte_size: self.total_byte_size,
- schema_descr: self.schema_descr,
- })
- }
-}
-
-/// Metadata for a column chunk.
-#[derive(Debug, Clone)]
-pub struct ColumnChunkMetaData {
- column_type: Type,
- column_path: ColumnPath,
- column_descr: ColumnDescPtr,
- encodings: Vec<Encoding>,
- file_path: Option<String>,
- file_offset: i64,
- num_values: i64,
- compression: Compression,
- total_compressed_size: i64,
- total_uncompressed_size: i64,
- data_page_offset: i64,
- index_page_offset: Option<i64>,
- dictionary_page_offset: Option<i64>,
- statistics: Option<Statistics>,
-}
-
-/// Represents common operations for a column chunk.
-impl ColumnChunkMetaData {
- /// Returns builder for column chunk metadata.
- pub fn builder(column_descr: ColumnDescPtr) -> ColumnChunkMetaDataBuilder {
- ColumnChunkMetaDataBuilder::new(column_descr)
- }
-
- /// File where the column chunk is stored.
- ///
- /// If not set, assumed to belong to the same file as the metadata.
- /// This path is relative to the current file.
- pub fn file_path(&self) -> Option<&String> {
- self.file_path.as_ref()
- }
-
- /// Byte offset in `file_path()`.
- pub fn file_offset(&self) -> i64 {
- self.file_offset
- }
-
- /// Type of this column. Must be primitive.
- pub fn column_type(&self) -> Type {
- self.column_type
- }
-
- /// Path (or identifier) of this column.
- pub fn column_path(&self) -> &ColumnPath {
- &self.column_path
- }
-
- /// Descriptor for this column.
- pub fn column_descr(&self) -> &ColumnDescriptor {
- self.column_descr.as_ref()
- }
-
- /// Reference counted clone of descriptor for this column.
- pub fn column_descr_ptr(&self) -> ColumnDescPtr {
- self.column_descr.clone()
- }
-
- /// All encodings used for this column.
- pub fn encodings(&self) -> &Vec<Encoding> {
- &self.encodings
- }
-
- /// Total number of values in this column chunk.
- pub fn num_values(&self) -> i64 {
- self.num_values
- }
-
- /// Compression for this column.
- pub fn compression(&self) -> Compression {
- self.compression
- }
-
- /// Returns the total compressed data size of this column chunk.
- pub fn compressed_size(&self) -> i64 {
- self.total_compressed_size
- }
-
- /// Returns the total uncompressed data size of this column chunk.
- pub fn uncompressed_size(&self) -> i64 {
- self.total_uncompressed_size
- }
-
- /// Returns the offset for the column data.
- pub fn data_page_offset(&self) -> i64 {
- self.data_page_offset
- }
-
- /// Returns `true` if this column chunk contains a index page, `false` otherwise.
- pub fn has_index_page(&self) -> bool {
- self.index_page_offset.is_some()
- }
-
- /// Returns the offset for the index page.
- pub fn index_page_offset(&self) -> Option<i64> {
- self.index_page_offset
- }
-
- /// Returns `true` if this column chunk contains a dictionary page, `false` otherwise.
- pub fn has_dictionary_page(&self) -> bool {
- self.dictionary_page_offset.is_some()
- }
-
- /// Returns the offset for the dictionary page, if any.
- pub fn dictionary_page_offset(&self) -> Option<i64> {
- self.dictionary_page_offset
- }
-
- /// Returns the offset and length in bytes of the column chunk within the file
- pub fn byte_range(&self) -> (u64, u64) {
- let col_start = if self.has_dictionary_page() {
- self.dictionary_page_offset().unwrap()
- } else {
- self.data_page_offset()
- };
- let col_len = self.compressed_size();
- assert!(
- col_start >= 0 && col_len >= 0,
- "column start and length should not be negative"
- );
- (col_start as u64, col_len as u64)
- }
-
- /// Returns statistics that are set for this column chunk,
- /// or `None` if no statistics are available.
- pub fn statistics(&self) -> Option<&Statistics> {
- self.statistics.as_ref()
- }
-
- /// Method to convert from Thrift.
- pub fn from_thrift(column_descr: ColumnDescPtr, cc: ColumnChunk) -> Result<Self> {
- if cc.meta_data.is_none() {
- return Err(general_err!("Expected to have column metadata"));
- }
- let mut col_metadata: ColumnMetaData = cc.meta_data.unwrap();
- let column_type = Type::from(col_metadata.type_);
- let column_path = ColumnPath::new(col_metadata.path_in_schema);
- let encodings = col_metadata
- .encodings
- .drain(0..)
- .map(Encoding::from)
- .collect();
- let compression = Compression::from(col_metadata.codec);
- let file_path = cc.file_path;
- let file_offset = cc.file_offset;
- let num_values = col_metadata.num_values;
- let total_compressed_size = col_metadata.total_compressed_size;
- let total_uncompressed_size = col_metadata.total_uncompressed_size;
- let data_page_offset = col_metadata.data_page_offset;
- let index_page_offset = col_metadata.index_page_offset;
- let dictionary_page_offset = col_metadata.dictionary_page_offset;
- let statistics = statistics::from_thrift(column_type, col_metadata.statistics);
- let result = ColumnChunkMetaData {
- column_type,
- column_path,
- column_descr,
- encodings,
- file_path,
- file_offset,
- num_values,
- compression,
- total_compressed_size,
- total_uncompressed_size,
- data_page_offset,
- index_page_offset,
- dictionary_page_offset,
- statistics,
- };
- Ok(result)
- }
-
- /// Method to convert to Thrift.
- pub fn to_thrift(&self) -> ColumnChunk {
- let column_metadata = ColumnMetaData {
- type_: self.column_type.into(),
- encodings: self.encodings().iter().map(|&v| v.into()).collect(),
- path_in_schema: Vec::from(self.column_path.as_ref()),
- codec: self.compression.into(),
- num_values: self.num_values,
- total_uncompressed_size: self.total_uncompressed_size,
- total_compressed_size: self.total_compressed_size,
- key_value_metadata: None,
- data_page_offset: self.data_page_offset,
- index_page_offset: self.index_page_offset,
- dictionary_page_offset: self.dictionary_page_offset,
- statistics: statistics::to_thrift(self.statistics.as_ref()),
- encoding_stats: None,
- };
-
- ColumnChunk {
- file_path: self.file_path().cloned(),
- file_offset: self.file_offset,
- meta_data: Some(column_metadata),
- offset_index_offset: None,
- offset_index_length: None,
- column_index_offset: None,
- column_index_length: None,
- }
- }
-}
-
-/// Builder for column chunk metadata.
-pub struct ColumnChunkMetaDataBuilder {
- column_descr: ColumnDescPtr,
- encodings: Vec<Encoding>,
- file_path: Option<String>,
- file_offset: i64,
- num_values: i64,
- compression: Compression,
- total_compressed_size: i64,
- total_uncompressed_size: i64,
- data_page_offset: i64,
- index_page_offset: Option<i64>,
- dictionary_page_offset: Option<i64>,
- statistics: Option<Statistics>,
-}
-
-impl ColumnChunkMetaDataBuilder {
- /// Creates new column chunk metadata builder.
- fn new(column_descr: ColumnDescPtr) -> Self {
- Self {
- column_descr,
- encodings: Vec::new(),
- file_path: None,
- file_offset: 0,
- num_values: 0,
- compression: Compression::UNCOMPRESSED,
- total_compressed_size: 0,
- total_uncompressed_size: 0,
- data_page_offset: 0,
- index_page_offset: None,
- dictionary_page_offset: None,
- statistics: None,
- }
- }
-
- /// Sets list of encodings for this column chunk.
- pub fn set_encodings(mut self, encodings: Vec<Encoding>) -> Self {
- self.encodings = encodings;
- self
- }
-
- /// Sets optional file path for this column chunk.
- pub fn set_file_path(mut self, value: String) -> Self {
- self.file_path = Some(value);
- self
- }
-
- /// Sets file offset in bytes.
- pub fn set_file_offset(mut self, value: i64) -> Self {
- self.file_offset = value;
- self
- }
-
- /// Sets number of values.
- pub fn set_num_values(mut self, value: i64) -> Self {
- self.num_values = value;
- self
- }
-
- /// Sets compression.
- pub fn set_compression(mut self, value: Compression) -> Self {
- self.compression = value;
- self
- }
-
- /// Sets total compressed size in bytes.
- pub fn set_total_compressed_size(mut self, value: i64) -> Self {
- self.total_compressed_size = value;
- self
- }
-
- /// Sets total uncompressed size in bytes.
- pub fn set_total_uncompressed_size(mut self, value: i64) -> Self {
- self.total_uncompressed_size = value;
- self
- }
-
- /// Sets data page offset in bytes.
- pub fn set_data_page_offset(mut self, value: i64) -> Self {
- self.data_page_offset = value;
- self
- }
-
- /// Sets optional dictionary page ofset in bytes.
- pub fn set_dictionary_page_offset(mut self, value: Option<i64>) -> Self {
- self.dictionary_page_offset = value;
- self
- }
-
- /// Sets optional index page offset in bytes.
- pub fn set_index_page_offset(mut self, value: Option<i64>) -> Self {
- self.index_page_offset = value;
- self
- }
-
- /// Sets statistics for this column chunk.
- pub fn set_statistics(mut self, value: Statistics) -> Self {
- self.statistics = Some(value);
- self
- }
-
- /// Builds column chunk metadata.
- pub fn build(self) -> Result<ColumnChunkMetaData> {
- Ok(ColumnChunkMetaData {
- column_type: self.column_descr.physical_type(),
- column_path: self.column_descr.path().clone(),
- column_descr: self.column_descr,
- encodings: self.encodings,
- file_path: self.file_path,
- file_offset: self.file_offset,
- num_values: self.num_values,
- compression: self.compression,
- total_compressed_size: self.total_compressed_size,
- total_uncompressed_size: self.total_uncompressed_size,
- data_page_offset: self.data_page_offset,
- index_page_offset: self.index_page_offset,
- dictionary_page_offset: self.dictionary_page_offset,
- statistics: self.statistics,
- })
- }
-}
-
-#[cfg(test)]
-mod tests {
- use super::*;
-
- #[test]
- fn test_row_group_metadata_thrift_conversion() {
- let schema_descr = get_test_schema_descr();
-
- let mut columns = vec![];
- for ptr in schema_descr.columns() {
- let column = ColumnChunkMetaData::builder(ptr.clone()).build().unwrap();
- columns.push(column);
- }
- let row_group_meta = RowGroupMetaData::builder(schema_descr.clone())
- .set_num_rows(1000)
- .set_total_byte_size(2000)
- .set_column_metadata(columns)
- .build()
- .unwrap();
-
- let row_group_exp = row_group_meta.to_thrift();
- let row_group_res =
- RowGroupMetaData::from_thrift(schema_descr, row_group_exp.clone())
- .unwrap()
- .to_thrift();
-
- assert_eq!(row_group_res, row_group_exp);
- }
-
- #[test]
- fn test_row_group_metadata_thrift_conversion_empty() {
- let schema_descr = get_test_schema_descr();
-
- let row_group_meta = RowGroupMetaData::builder(schema_descr).build();
-
- assert!(row_group_meta.is_err());
- if let Err(e) = row_group_meta {
- assert_eq!(
- format!("{}", e),
- "Parquet error: Column length mismatch: 2 != 0"
- );
- }
- }
-
- #[test]
- fn test_column_chunk_metadata_thrift_conversion() {
- let column_descr = get_test_schema_descr().column(0);
-
- let col_metadata = ColumnChunkMetaData::builder(column_descr.clone())
- .set_encodings(vec![Encoding::PLAIN, Encoding::RLE])
- .set_file_path("file_path".to_owned())
- .set_file_offset(100)
- .set_num_values(1000)
- .set_compression(Compression::SNAPPY)
- .set_total_compressed_size(2000)
- .set_total_uncompressed_size(3000)
- .set_data_page_offset(4000)
- .set_dictionary_page_offset(Some(5000))
- .build()
- .unwrap();
-
- let col_chunk_exp = col_metadata.to_thrift();
-
- let col_chunk_res =
- ColumnChunkMetaData::from_thrift(column_descr, col_chunk_exp.clone())
- .unwrap()
- .to_thrift();
-
- assert_eq!(col_chunk_res, col_chunk_exp);
- }
-
- #[test]
- fn test_column_chunk_metadata_thrift_conversion_empty() {
- let column_descr = get_test_schema_descr().column(0);
-
- let col_metadata = ColumnChunkMetaData::builder(column_descr.clone())
- .build()
- .unwrap();
-
- let col_chunk_exp = col_metadata.to_thrift();
- let col_chunk_res =
- ColumnChunkMetaData::from_thrift(column_descr, col_chunk_exp.clone())
- .unwrap()
- .to_thrift();
-
- assert_eq!(col_chunk_res, col_chunk_exp);
- }
-
- #[test]
- fn test_compressed_size() {
- let schema_descr = get_test_schema_descr();
-
- let mut columns = vec![];
- for column_descr in schema_descr.columns() {
- let column = ColumnChunkMetaData::builder(column_descr.clone())
- .set_total_compressed_size(500)
- .set_total_uncompressed_size(700)
- .build()
- .unwrap();
- columns.push(column);
- }
- let row_group_meta = RowGroupMetaData::builder(schema_descr)
- .set_num_rows(1000)
- .set_column_metadata(columns)
- .build()
- .unwrap();
-
- let compressed_size_res: i64 = row_group_meta.compressed_size();
- let compressed_size_exp: i64 = 1000;
-
- assert_eq!(compressed_size_res, compressed_size_exp);
- }
-
- /// Returns sample schema descriptor so we can create column metadata.
- fn get_test_schema_descr() -> SchemaDescPtr {
- let schema = SchemaType::group_type_builder("schema")
- .with_fields(&mut vec![
- Arc::new(
- SchemaType::primitive_type_builder("a", Type::INT32)
- .build()
- .unwrap(),
- ),
- Arc::new(
- SchemaType::primitive_type_builder("b", Type::INT32)
- .build()
- .unwrap(),
- ),
- ])
- .build()
- .unwrap();
-
- Arc::new(SchemaDescriptor::new(Arc::new(schema)))
- }
-}
diff --git a/parquet/src/file/mod.rs b/parquet/src/file/mod.rs
deleted file mode 100644
index f85de98..0000000
--- a/parquet/src/file/mod.rs
+++ /dev/null
@@ -1,110 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-//! Main entrypoint for working with Parquet API.
-//!
-//! Provides access to file and row group readers and writers, record API, metadata, etc.
-//!
-//! See [`reader::SerializedFileReader`](reader/struct.SerializedFileReader.html) or
-//! [`writer::SerializedFileWriter`](writer/struct.SerializedFileWriter.html) for a
-//! starting reference, [`metadata::ParquetMetaData`](metadata/index.html) for file
-//! metadata, and [`statistics`](statistics/index.html) for working with statistics.
-//!
-//! # Example of writing a new file
-//!
-//! ```rust,no_run
-//! use std::{fs, path::Path, sync::Arc};
-//!
-//! use parquet::{
-//! file::{
-//! properties::WriterProperties,
-//! writer::{FileWriter, SerializedFileWriter},
-//! },
-//! schema::parser::parse_message_type,
-//! };
-//!
-//! let path = Path::new("/path/to/sample.parquet");
-//!
-//! let message_type = "
-//! message schema {
-//! REQUIRED INT32 b;
-//! }
-//! ";
-//! let schema = Arc::new(parse_message_type(message_type).unwrap());
-//! let props = Arc::new(WriterProperties::builder().build());
-//! let file = fs::File::create(&path).unwrap();
-//! let mut writer = SerializedFileWriter::new(file, schema, props).unwrap();
-//! let mut row_group_writer = writer.next_row_group().unwrap();
-//! while let Some(mut col_writer) = row_group_writer.next_column().unwrap() {
-//! // ... write values to a column writer
-//! row_group_writer.close_column(col_writer).unwrap();
-//! }
-//! writer.close_row_group(row_group_writer).unwrap();
-//! writer.close().unwrap();
-//!
-//! let bytes = fs::read(&path).unwrap();
-//! assert_eq!(&bytes[0..4], &[b'P', b'A', b'R', b'1']);
-//! ```
-//! # Example of reading an existing file
-//!
-//! ```rust,no_run
-//! use parquet::file::reader::{FileReader, SerializedFileReader};
-//! use std::{fs::File, path::Path};
-//!
-//! let path = Path::new("/path/to/sample.parquet");
-//! if let Ok(file) = File::open(&path) {
-//! let reader = SerializedFileReader::new(file).unwrap();
-//!
-//! let parquet_metadata = reader.metadata();
-//! assert_eq!(parquet_metadata.num_row_groups(), 1);
-//!
-//! let row_group_reader = reader.get_row_group(0).unwrap();
-//! assert_eq!(row_group_reader.num_columns(), 1);
-//! }
-//! ```
-//! # Example of reading multiple files
-//!
-//! ```rust,no_run
-//! use parquet::file::reader::SerializedFileReader;
-//! use std::convert::TryFrom;
-//!
-//! let paths = vec![
-//! "/path/to/sample.parquet/part-1.snappy.parquet",
-//! "/path/to/sample.parquet/part-2.snappy.parquet"
-//! ];
-//! // Create a reader for each file and flat map rows
-//! let rows = paths.iter()
-//! .map(|p| SerializedFileReader::try_from(*p).unwrap())
-//! .flat_map(|r| r.into_iter());
-//!
-//! for row in rows {
-//! println!("{}", row);
-//! }
-//! ```
-pub mod footer;
-pub mod metadata;
-pub mod properties;
-pub mod reader;
-pub mod serialized_reader;
-pub mod statistics;
-pub mod writer;
-
-const FOOTER_SIZE: usize = 8;
-const PARQUET_MAGIC: [u8; 4] = [b'P', b'A', b'R', b'1'];
-
-/// The number of bytes read at the end of the parquet file on first read
-const DEFAULT_FOOTER_READ_SIZE: usize = 64 * 1024;
diff --git a/parquet/src/file/properties.rs b/parquet/src/file/properties.rs
deleted file mode 100644
index 0d0cbef..0000000
--- a/parquet/src/file/properties.rs
+++ /dev/null
@@ -1,680 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-//! Writer properties.
-//!
-//! # Usage
-//!
-//! ```rust
-//! use parquet::{
-//! basic::{Compression, Encoding},
-//! file::properties::*,
-//! schema::types::ColumnPath,
-//! };
-//!
-//! // Create properties with default configuration.
-//! let props = WriterProperties::builder().build();
-//!
-//! // Use properties builder to set certain options and assemble the configuration.
-//! let props = WriterProperties::builder()
-//! .set_writer_version(WriterVersion::PARQUET_1_0)
-//! .set_encoding(Encoding::PLAIN)
-//! .set_column_encoding(ColumnPath::from("col1"), Encoding::DELTA_BINARY_PACKED)
-//! .set_compression(Compression::SNAPPY)
-//! .build();
-//!
-//! assert_eq!(props.writer_version(), WriterVersion::PARQUET_1_0);
-//! assert_eq!(
-//! props.encoding(&ColumnPath::from("col1")),
-//! Some(Encoding::DELTA_BINARY_PACKED)
-//! );
-//! assert_eq!(
-//! props.encoding(&ColumnPath::from("col2")),
-//! Some(Encoding::PLAIN)
-//! );
-//! ```
-
-use std::{collections::HashMap, sync::Arc};
-
-use crate::basic::{Compression, Encoding};
-use crate::file::metadata::KeyValue;
-use crate::schema::types::ColumnPath;
-
-const DEFAULT_PAGE_SIZE: usize = 1024 * 1024;
-const DEFAULT_WRITE_BATCH_SIZE: usize = 1024;
-const DEFAULT_WRITER_VERSION: WriterVersion = WriterVersion::PARQUET_1_0;
-const DEFAULT_COMPRESSION: Compression = Compression::UNCOMPRESSED;
-const DEFAULT_DICTIONARY_ENABLED: bool = true;
-const DEFAULT_DICTIONARY_PAGE_SIZE_LIMIT: usize = DEFAULT_PAGE_SIZE;
-const DEFAULT_STATISTICS_ENABLED: bool = true;
-const DEFAULT_MAX_STATISTICS_SIZE: usize = 4096;
-const DEFAULT_MAX_ROW_GROUP_SIZE: usize = 128 * 1024 * 1024;
-const DEFAULT_CREATED_BY: &str = env!("PARQUET_CREATED_BY");
-
-/// Parquet writer version.
-///
-/// Basic constant, which is not part of the Thrift definition.
-#[derive(Debug, Clone, Copy, PartialEq)]
-pub enum WriterVersion {
- PARQUET_1_0,
- PARQUET_2_0,
-}
-
-impl WriterVersion {
- /// Returns writer version as `i32`.
- pub fn as_num(&self) -> i32 {
- match self {
- WriterVersion::PARQUET_1_0 => 1,
- WriterVersion::PARQUET_2_0 => 2,
- }
- }
-}
-
-/// Reference counted writer properties.
-pub type WriterPropertiesPtr = Arc<WriterProperties>;
-
-/// Writer properties.
-///
-/// All properties except the key-value metadata are immutable,
-/// use [`WriterPropertiesBuilder`] to assemble these properties.
-#[derive(Debug, Clone)]
-pub struct WriterProperties {
- data_pagesize_limit: usize,
- dictionary_pagesize_limit: usize,
- write_batch_size: usize,
- max_row_group_size: usize,
- writer_version: WriterVersion,
- created_by: String,
- pub(crate) key_value_metadata: Option<Vec<KeyValue>>,
- default_column_properties: ColumnProperties,
- column_properties: HashMap<ColumnPath, ColumnProperties>,
-}
-
-impl WriterProperties {
- /// Returns builder for writer properties with default values.
- pub fn builder() -> WriterPropertiesBuilder {
- WriterPropertiesBuilder::with_defaults()
- }
-
- /// Returns data page size limit.
- pub fn data_pagesize_limit(&self) -> usize {
- self.data_pagesize_limit
- }
-
- /// Returns dictionary page size limit.
- pub fn dictionary_pagesize_limit(&self) -> usize {
- self.dictionary_pagesize_limit
- }
-
- /// Returns configured batch size for writes.
- ///
- /// When writing a batch of data, this setting allows to split it internally into
- /// smaller batches so we can better estimate the size of a page currently being
- /// written.
- pub fn write_batch_size(&self) -> usize {
- self.write_batch_size
- }
-
- /// Returns max size for a row group.
- pub fn max_row_group_size(&self) -> usize {
- self.max_row_group_size
- }
-
- /// Returns configured writer version.
- pub fn writer_version(&self) -> WriterVersion {
- self.writer_version
- }
-
- /// Returns `created_by` string.
- pub fn created_by(&self) -> &str {
- &self.created_by
- }
-
- /// Returns `key_value_metadata` KeyValue pairs.
- pub fn key_value_metadata(&self) -> &Option<Vec<KeyValue>> {
- &self.key_value_metadata
- }
-
- /// Returns encoding for a data page, when dictionary encoding is enabled.
- /// This is not configurable.
- #[inline]
- pub fn dictionary_data_page_encoding(&self) -> Encoding {
- // PLAIN_DICTIONARY encoding is deprecated in writer version 1.
- // Dictionary values are encoded using RLE_DICTIONARY encoding.
- Encoding::RLE_DICTIONARY
- }
-
- /// Returns encoding for dictionary page, when dictionary encoding is enabled.
- /// This is not configurable.
- #[inline]
- pub fn dictionary_page_encoding(&self) -> Encoding {
- // PLAIN_DICTIONARY is deprecated in writer version 1.
- // Dictionary is encoded using plain encoding.
- Encoding::PLAIN
- }
-
- /// Returns encoding for a column, if set.
- /// In case when dictionary is enabled, returns fallback encoding.
- ///
- /// If encoding is not set, then column writer will choose the best encoding
- /// based on the column type.
- pub fn encoding(&self, col: &ColumnPath) -> Option<Encoding> {
- self.column_properties
- .get(col)
- .and_then(|c| c.encoding())
- .or_else(|| self.default_column_properties.encoding())
- }
-
- /// Returns compression codec for a column.
- pub fn compression(&self, col: &ColumnPath) -> Compression {
- self.column_properties
- .get(col)
- .and_then(|c| c.compression())
- .or_else(|| self.default_column_properties.compression())
- .unwrap_or(DEFAULT_COMPRESSION)
- }
-
- /// Returns `true` if dictionary encoding is enabled for a column.
- pub fn dictionary_enabled(&self, col: &ColumnPath) -> bool {
- self.column_properties
- .get(col)
- .and_then(|c| c.dictionary_enabled())
- .or_else(|| self.default_column_properties.dictionary_enabled())
- .unwrap_or(DEFAULT_DICTIONARY_ENABLED)
- }
-
- /// Returns `true` if statistics are enabled for a column.
- pub fn statistics_enabled(&self, col: &ColumnPath) -> bool {
- self.column_properties
- .get(col)
- .and_then(|c| c.statistics_enabled())
- .or_else(|| self.default_column_properties.statistics_enabled())
- .unwrap_or(DEFAULT_STATISTICS_ENABLED)
- }
-
- /// Returns max size for statistics.
- /// Only applicable if statistics are enabled.
- pub fn max_statistics_size(&self, col: &ColumnPath) -> usize {
- self.column_properties
- .get(col)
- .and_then(|c| c.max_statistics_size())
- .or_else(|| self.default_column_properties.max_statistics_size())
- .unwrap_or(DEFAULT_MAX_STATISTICS_SIZE)
- }
-}
-
-/// Writer properties builder.
-pub struct WriterPropertiesBuilder {
- data_pagesize_limit: usize,
- dictionary_pagesize_limit: usize,
- write_batch_size: usize,
- max_row_group_size: usize,
- writer_version: WriterVersion,
- created_by: String,
- key_value_metadata: Option<Vec<KeyValue>>,
- default_column_properties: ColumnProperties,
- column_properties: HashMap<ColumnPath, ColumnProperties>,
-}
-
-impl WriterPropertiesBuilder {
- /// Returns default state of the builder.
- fn with_defaults() -> Self {
- Self {
- data_pagesize_limit: DEFAULT_PAGE_SIZE,
- dictionary_pagesize_limit: DEFAULT_DICTIONARY_PAGE_SIZE_LIMIT,
- write_batch_size: DEFAULT_WRITE_BATCH_SIZE,
- max_row_group_size: DEFAULT_MAX_ROW_GROUP_SIZE,
- writer_version: DEFAULT_WRITER_VERSION,
- created_by: DEFAULT_CREATED_BY.to_string(),
- key_value_metadata: None,
- default_column_properties: ColumnProperties::new(),
- column_properties: HashMap::new(),
- }
- }
-
- /// Finalizes the configuration and returns immutable writer properties struct.
- pub fn build(self) -> WriterProperties {
- WriterProperties {
- data_pagesize_limit: self.data_pagesize_limit,
- dictionary_pagesize_limit: self.dictionary_pagesize_limit,
- write_batch_size: self.write_batch_size,
- max_row_group_size: self.max_row_group_size,
- writer_version: self.writer_version,
- created_by: self.created_by,
- key_value_metadata: self.key_value_metadata,
- default_column_properties: self.default_column_properties,
- column_properties: self.column_properties,
- }
- }
-
- // ----------------------------------------------------------------------
- // Writer properties related to a file
-
- /// Sets writer version.
- pub fn set_writer_version(mut self, value: WriterVersion) -> Self {
- self.writer_version = value;
- self
- }
-
- /// Sets data page size limit.
- pub fn set_data_pagesize_limit(mut self, value: usize) -> Self {
- self.data_pagesize_limit = value;
- self
- }
-
- /// Sets dictionary page size limit.
- pub fn set_dictionary_pagesize_limit(mut self, value: usize) -> Self {
- self.dictionary_pagesize_limit = value;
- self
- }
-
- /// Sets write batch size.
- pub fn set_write_batch_size(mut self, value: usize) -> Self {
- self.write_batch_size = value;
- self
- }
-
- /// Sets max size for a row group.
- pub fn set_max_row_group_size(mut self, value: usize) -> Self {
- assert!(value > 0, "Cannot have a 0 max row group size");
- self.max_row_group_size = value;
- self
- }
-
- /// Sets "created by" property.
- pub fn set_created_by(mut self, value: String) -> Self {
- self.created_by = value;
- self
- }
-
- /// Sets "key_value_metadata" property.
- pub fn set_key_value_metadata(mut self, value: Option<Vec<KeyValue>>) -> Self {
- self.key_value_metadata = value;
- self
- }
-
- // ----------------------------------------------------------------------
- // Setters for any column (global)
-
- /// Sets encoding for any column.
- ///
- /// If dictionary is not enabled, this is treated as a primary encoding for all
- /// columns. In case when dictionary is enabled for any column, this value is
- /// considered to be a fallback encoding for that column.
- ///
- /// Panics if user tries to set dictionary encoding here, regardless of dictionary
- /// encoding flag being set.
- pub fn set_encoding(mut self, value: Encoding) -> Self {
- self.default_column_properties.set_encoding(value);
- self
- }
-
- /// Sets compression codec for any column.
- pub fn set_compression(mut self, value: Compression) -> Self {
- self.default_column_properties.set_compression(value);
- self
- }
-
- /// Sets flag to enable/disable dictionary encoding for any column.
- ///
- /// Use this method to set dictionary encoding, instead of explicitly specifying
- /// encoding in `set_encoding` method.
- pub fn set_dictionary_enabled(mut self, value: bool) -> Self {
- self.default_column_properties.set_dictionary_enabled(value);
- self
- }
-
- /// Sets flag to enable/disable statistics for any column.
- pub fn set_statistics_enabled(mut self, value: bool) -> Self {
- self.default_column_properties.set_statistics_enabled(value);
- self
- }
-
- /// Sets max statistics size for any column.
- /// Applicable only if statistics are enabled.
- pub fn set_max_statistics_size(mut self, value: usize) -> Self {
- self.default_column_properties
- .set_max_statistics_size(value);
- self
- }
-
- // ----------------------------------------------------------------------
- // Setters for a specific column
-
- /// Helper method to get existing or new mutable reference of column properties.
- #[inline]
- fn get_mut_props(&mut self, col: ColumnPath) -> &mut ColumnProperties {
- self.column_properties
- .entry(col)
- .or_insert(ColumnProperties::new())
- }
-
- /// Sets encoding for a column.
- /// Takes precedence over globally defined settings.
- ///
- /// If dictionary is not enabled, this is treated as a primary encoding for this
- /// column. In case when dictionary is enabled for this column, either through
- /// global defaults or explicitly, this value is considered to be a fallback
- /// encoding for this column.
- ///
- /// Panics if user tries to set dictionary encoding here, regardless of dictionary
- /// encoding flag being set.
- pub fn set_column_encoding(mut self, col: ColumnPath, value: Encoding) -> Self {
- self.get_mut_props(col).set_encoding(value);
- self
- }
-
- /// Sets compression codec for a column.
- /// Takes precedence over globally defined settings.
- pub fn set_column_compression(mut self, col: ColumnPath, value: Compression) -> Self {
- self.get_mut_props(col).set_compression(value);
- self
- }
-
- /// Sets flag to enable/disable dictionary encoding for a column.
- /// Takes precedence over globally defined settings.
- pub fn set_column_dictionary_enabled(mut self, col: ColumnPath, value: bool) -> Self {
- self.get_mut_props(col).set_dictionary_enabled(value);
- self
- }
-
- /// Sets flag to enable/disable statistics for a column.
- /// Takes precedence over globally defined settings.
- pub fn set_column_statistics_enabled(mut self, col: ColumnPath, value: bool) -> Self {
- self.get_mut_props(col).set_statistics_enabled(value);
- self
- }
-
- /// Sets max size for statistics for a column.
- /// Takes precedence over globally defined settings.
- pub fn set_column_max_statistics_size(
- mut self,
- col: ColumnPath,
- value: usize,
- ) -> Self {
- self.get_mut_props(col).set_max_statistics_size(value);
- self
- }
-}
-
-/// Container for column properties that can be changed as part of writer.
-///
-/// If a field is `None`, it means that no specific value has been set for this column,
-/// so some subsequent or default value must be used.
-#[derive(Debug, Clone, PartialEq)]
-struct ColumnProperties {
- encoding: Option<Encoding>,
- codec: Option<Compression>,
- dictionary_enabled: Option<bool>,
- statistics_enabled: Option<bool>,
- max_statistics_size: Option<usize>,
-}
-
-impl ColumnProperties {
- /// Initialise column properties with default values.
- fn new() -> Self {
- Self {
- encoding: None,
- codec: None,
- dictionary_enabled: None,
- statistics_enabled: None,
- max_statistics_size: None,
- }
- }
-
- /// Sets encoding for this column.
- ///
- /// If dictionary is not enabled, this is treated as a primary encoding for a column.
- /// In case when dictionary is enabled for a column, this value is considered to
- /// be a fallback encoding.
- ///
- /// Panics if user tries to set dictionary encoding here, regardless of dictionary
- /// encoding flag being set. Use `set_dictionary_enabled` method to enable dictionary
- /// for a column.
- fn set_encoding(&mut self, value: Encoding) {
- if value == Encoding::PLAIN_DICTIONARY || value == Encoding::RLE_DICTIONARY {
- panic!("Dictionary encoding can not be used as fallback encoding");
- }
- self.encoding = Some(value);
- }
-
- /// Sets compression codec for this column.
- fn set_compression(&mut self, value: Compression) {
- self.codec = Some(value);
- }
-
- /// Sets whether or not dictionary encoding is enabled for this column.
- fn set_dictionary_enabled(&mut self, enabled: bool) {
- self.dictionary_enabled = Some(enabled);
- }
-
- /// Sets whether or not statistics are enabled for this column.
- fn set_statistics_enabled(&mut self, enabled: bool) {
- self.statistics_enabled = Some(enabled);
- }
-
- /// Sets max size for statistics for this column.
- fn set_max_statistics_size(&mut self, value: usize) {
- self.max_statistics_size = Some(value);
- }
-
- /// Returns optional encoding for this column.
- fn encoding(&self) -> Option<Encoding> {
- self.encoding
- }
-
- /// Returns optional compression codec for this column.
- fn compression(&self) -> Option<Compression> {
- self.codec
- }
-
- /// Returns `Some(true)` if dictionary encoding is enabled for this column, if
- /// disabled then returns `Some(false)`. If result is `None`, then no setting has
- /// been provided.
- fn dictionary_enabled(&self) -> Option<bool> {
- self.dictionary_enabled
- }
-
- /// Returns `Some(true)` if statistics are enabled for this column, if disabled then
- /// returns `Some(false)`. If result is `None`, then no setting has been provided.
- fn statistics_enabled(&self) -> Option<bool> {
- self.statistics_enabled
- }
-
- /// Returns optional max size in bytes for statistics.
- fn max_statistics_size(&self) -> Option<usize> {
- self.max_statistics_size
- }
-}
-
-#[cfg(test)]
-mod tests {
- use super::*;
-
- #[test]
- fn test_writer_version() {
- assert_eq!(WriterVersion::PARQUET_1_0.as_num(), 1);
- assert_eq!(WriterVersion::PARQUET_2_0.as_num(), 2);
- }
-
- #[test]
- fn test_writer_properties_default_settings() {
- let props = WriterProperties::builder().build();
- assert_eq!(props.data_pagesize_limit(), DEFAULT_PAGE_SIZE);
- assert_eq!(
- props.dictionary_pagesize_limit(),
- DEFAULT_DICTIONARY_PAGE_SIZE_LIMIT
- );
- assert_eq!(props.write_batch_size(), DEFAULT_WRITE_BATCH_SIZE);
- assert_eq!(props.max_row_group_size(), DEFAULT_MAX_ROW_GROUP_SIZE);
- assert_eq!(props.writer_version(), DEFAULT_WRITER_VERSION);
- assert_eq!(props.created_by(), DEFAULT_CREATED_BY);
- assert_eq!(props.key_value_metadata(), &None);
- assert_eq!(props.encoding(&ColumnPath::from("col")), None);
- assert_eq!(
- props.compression(&ColumnPath::from("col")),
- DEFAULT_COMPRESSION
- );
- assert_eq!(
- props.dictionary_enabled(&ColumnPath::from("col")),
- DEFAULT_DICTIONARY_ENABLED
- );
- assert_eq!(
- props.statistics_enabled(&ColumnPath::from("col")),
- DEFAULT_STATISTICS_ENABLED
- );
- assert_eq!(
- props.max_statistics_size(&ColumnPath::from("col")),
- DEFAULT_MAX_STATISTICS_SIZE
- );
- }
-
- #[test]
- fn test_writer_properties_dictionary_encoding() {
- // dictionary encoding is not configurable, and it should be the same for both
- // writer version 1 and 2.
- for version in &[WriterVersion::PARQUET_1_0, WriterVersion::PARQUET_2_0] {
- let props = WriterProperties::builder()
- .set_writer_version(*version)
- .build();
- assert_eq!(props.dictionary_page_encoding(), Encoding::PLAIN);
- assert_eq!(
- props.dictionary_data_page_encoding(),
- Encoding::RLE_DICTIONARY
- );
- }
- }
-
- #[test]
- #[should_panic(expected = "Dictionary encoding can not be used as fallback encoding")]
- fn test_writer_properties_panic_when_plain_dictionary_is_fallback() {
- // Should panic when user specifies dictionary encoding as fallback encoding.
- WriterProperties::builder()
- .set_encoding(Encoding::PLAIN_DICTIONARY)
- .build();
- }
-
- #[test]
- #[should_panic(expected = "Dictionary encoding can not be used as fallback encoding")]
- fn test_writer_properties_panic_when_rle_dictionary_is_fallback() {
- // Should panic when user specifies dictionary encoding as fallback encoding.
- WriterProperties::builder()
- .set_encoding(Encoding::RLE_DICTIONARY)
- .build();
- }
-
- #[test]
- #[should_panic(expected = "Dictionary encoding can not be used as fallback encoding")]
- fn test_writer_properties_panic_when_dictionary_is_enabled() {
- WriterProperties::builder()
- .set_dictionary_enabled(true)
- .set_column_encoding(ColumnPath::from("col"), Encoding::RLE_DICTIONARY)
- .build();
- }
-
- #[test]
- #[should_panic(expected = "Dictionary encoding can not be used as fallback encoding")]
- fn test_writer_properties_panic_when_dictionary_is_disabled() {
- WriterProperties::builder()
- .set_dictionary_enabled(false)
- .set_column_encoding(ColumnPath::from("col"), Encoding::RLE_DICTIONARY)
- .build();
- }
-
- #[test]
- fn test_writer_properties_builder() {
- let props = WriterProperties::builder()
- // file settings
- .set_writer_version(WriterVersion::PARQUET_2_0)
- .set_data_pagesize_limit(10)
- .set_dictionary_pagesize_limit(20)
- .set_write_batch_size(30)
- .set_max_row_group_size(40)
- .set_created_by("default".to_owned())
- .set_key_value_metadata(Some(vec![KeyValue::new(
- "key".to_string(),
- "value".to_string(),
- )]))
- // global column settings
- .set_encoding(Encoding::DELTA_BINARY_PACKED)
- .set_compression(Compression::GZIP)
- .set_dictionary_enabled(false)
- .set_statistics_enabled(false)
- .set_max_statistics_size(50)
- // specific column settings
- .set_column_encoding(ColumnPath::from("col"), Encoding::RLE)
- .set_column_compression(ColumnPath::from("col"), Compression::SNAPPY)
- .set_column_dictionary_enabled(ColumnPath::from("col"), true)
- .set_column_statistics_enabled(ColumnPath::from("col"), true)
- .set_column_max_statistics_size(ColumnPath::from("col"), 123)
- .build();
-
- assert_eq!(props.writer_version(), WriterVersion::PARQUET_2_0);
- assert_eq!(props.data_pagesize_limit(), 10);
- assert_eq!(props.dictionary_pagesize_limit(), 20);
- assert_eq!(props.write_batch_size(), 30);
- assert_eq!(props.max_row_group_size(), 40);
- assert_eq!(props.created_by(), "default");
- assert_eq!(
- props.key_value_metadata(),
- &Some(vec![KeyValue::new("key".to_string(), "value".to_string(),)])
- );
-
- assert_eq!(
- props.encoding(&ColumnPath::from("a")),
- Some(Encoding::DELTA_BINARY_PACKED)
- );
- assert_eq!(props.compression(&ColumnPath::from("a")), Compression::GZIP);
- assert_eq!(props.dictionary_enabled(&ColumnPath::from("a")), false);
- assert_eq!(props.statistics_enabled(&ColumnPath::from("a")), false);
- assert_eq!(props.max_statistics_size(&ColumnPath::from("a")), 50);
-
- assert_eq!(
- props.encoding(&ColumnPath::from("col")),
- Some(Encoding::RLE)
- );
- assert_eq!(
- props.compression(&ColumnPath::from("col")),
- Compression::SNAPPY
- );
- assert_eq!(props.dictionary_enabled(&ColumnPath::from("col")), true);
- assert_eq!(props.statistics_enabled(&ColumnPath::from("col")), true);
- assert_eq!(props.max_statistics_size(&ColumnPath::from("col")), 123);
- }
-
- #[test]
- fn test_writer_properties_builder_partial_defaults() {
- let props = WriterProperties::builder()
- .set_encoding(Encoding::DELTA_BINARY_PACKED)
- .set_compression(Compression::GZIP)
- .set_column_encoding(ColumnPath::from("col"), Encoding::RLE)
- .build();
-
- assert_eq!(
- props.encoding(&ColumnPath::from("col")),
- Some(Encoding::RLE)
- );
- assert_eq!(
- props.compression(&ColumnPath::from("col")),
- Compression::GZIP
- );
- assert_eq!(
- props.dictionary_enabled(&ColumnPath::from("col")),
- DEFAULT_DICTIONARY_ENABLED
- );
- }
-}
diff --git a/parquet/src/file/reader.rs b/parquet/src/file/reader.rs
deleted file mode 100644
index aa8ba83..0000000
--- a/parquet/src/file/reader.rs
+++ /dev/null
@@ -1,206 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-//! Contains file reader API and provides methods to access file metadata, row group
-//! readers to read individual column chunks, or access record iterator.
-
-use std::{boxed::Box, io::Read, sync::Arc};
-
-use crate::column::page::PageIterator;
-use crate::column::{page::PageReader, reader::ColumnReader};
-use crate::errors::{ParquetError, Result};
-use crate::file::metadata::*;
-pub use crate::file::serialized_reader::{SerializedFileReader, SerializedPageReader};
-use crate::record::reader::RowIter;
-use crate::schema::types::{ColumnDescPtr, SchemaDescPtr, Type as SchemaType};
-
-use crate::basic::Type;
-
-use crate::column::reader::ColumnReaderImpl;
-
-/// Length should return the total number of bytes in the input source.
-/// It's mainly used to read the metadata, which is at the end of the source.
-#[allow(clippy::len_without_is_empty)]
-pub trait Length {
- /// Returns the amount of bytes of the inner source.
- fn len(&self) -> u64;
-}
-
-/// The ChunkReader trait generates readers of chunks of a source.
-/// For a file system reader, each chunk might contain a clone of File bounded on a given range.
-/// For an object store reader, each read can be mapped to a range request.
-pub trait ChunkReader: Length {
- type T: Read;
- /// get a serialy readeable slice of the current reader
- /// This should fail if the slice exceeds the current bounds
- fn get_read(&self, start: u64, length: usize) -> Result<Self::T>;
-}
-
-// ----------------------------------------------------------------------
-// APIs for file & row group readers
-
-/// Parquet file reader API. With this, user can get metadata information about the
-/// Parquet file, can get reader for each row group, and access record iterator.
-pub trait FileReader {
- /// Get metadata information about this file.
- fn metadata(&self) -> &ParquetMetaData;
-
- /// Get the total number of row groups for this file.
- fn num_row_groups(&self) -> usize;
-
- /// Get the `i`th row group reader. Note this doesn't do bound check.
- fn get_row_group(&self, i: usize) -> Result<Box<dyn RowGroupReader + '_>>;
-
- /// Get full iterator of `Row`s from a file (over all row groups).
- ///
- /// Iterator will automatically load the next row group to advance.
- ///
- /// Projected schema can be a subset of or equal to the file schema, when it is None,
- /// full file schema is assumed.
- fn get_row_iter(&self, projection: Option<SchemaType>) -> Result<RowIter>;
-}
-
-/// Parquet row group reader API. With this, user can get metadata information about the
-/// row group, as well as readers for each individual column chunk.
-pub trait RowGroupReader {
- /// Get metadata information about this row group.
- fn metadata(&self) -> &RowGroupMetaData;
-
- /// Get the total number of column chunks in this row group.
- fn num_columns(&self) -> usize;
-
- /// Get page reader for the `i`th column chunk.
- fn get_column_page_reader(&self, i: usize) -> Result<Box<dyn PageReader>>;
-
- /// Get value reader for the `i`th column chunk.
- fn get_column_reader(&self, i: usize) -> Result<ColumnReader> {
- let schema_descr = self.metadata().schema_descr();
- let col_descr = schema_descr.column(i);
- let col_page_reader = self.get_column_page_reader(i)?;
- let col_reader = match col_descr.physical_type() {
- Type::BOOLEAN => ColumnReader::BoolColumnReader(ColumnReaderImpl::new(
- col_descr,
- col_page_reader,
- )),
- Type::INT32 => ColumnReader::Int32ColumnReader(ColumnReaderImpl::new(
- col_descr,
- col_page_reader,
- )),
- Type::INT64 => ColumnReader::Int64ColumnReader(ColumnReaderImpl::new(
- col_descr,
- col_page_reader,
- )),
- Type::INT96 => ColumnReader::Int96ColumnReader(ColumnReaderImpl::new(
- col_descr,
- col_page_reader,
- )),
- Type::FLOAT => ColumnReader::FloatColumnReader(ColumnReaderImpl::new(
- col_descr,
- col_page_reader,
- )),
- Type::DOUBLE => ColumnReader::DoubleColumnReader(ColumnReaderImpl::new(
- col_descr,
- col_page_reader,
- )),
- Type::BYTE_ARRAY => ColumnReader::ByteArrayColumnReader(
- ColumnReaderImpl::new(col_descr, col_page_reader),
- ),
- Type::FIXED_LEN_BYTE_ARRAY => ColumnReader::FixedLenByteArrayColumnReader(
- ColumnReaderImpl::new(col_descr, col_page_reader),
- ),
- };
- Ok(col_reader)
- }
-
- /// Get iterator of `Row`s from this row group.
- ///
- /// Projected schema can be a subset of or equal to the file schema, when it is None,
- /// full file schema is assumed.
- fn get_row_iter(&self, projection: Option<SchemaType>) -> Result<RowIter>;
-}
-
-// ----------------------------------------------------------------------
-// Iterator
-
-/// Implementation of page iterator for parquet file.
-pub struct FilePageIterator {
- column_index: usize,
- row_group_indices: Box<dyn Iterator<Item = usize>>,
- file_reader: Arc<dyn FileReader>,
-}
-
-impl FilePageIterator {
- /// Creates a page iterator for all row groups in file.
- pub fn new(column_index: usize, file_reader: Arc<dyn FileReader>) -> Result<Self> {
- let num_row_groups = file_reader.metadata().num_row_groups();
-
- let row_group_indices = Box::new(0..num_row_groups);
-
- Self::with_row_groups(column_index, row_group_indices, file_reader)
- }
-
- /// Create page iterator from parquet file reader with only some row groups.
- pub fn with_row_groups(
- column_index: usize,
- row_group_indices: Box<dyn Iterator<Item = usize>>,
- file_reader: Arc<dyn FileReader>,
- ) -> Result<Self> {
- // Check that column_index is valid
- let num_columns = file_reader
- .metadata()
- .file_metadata()
- .schema_descr()
- .num_columns();
-
- if column_index >= num_columns {
- return Err(ParquetError::IndexOutOfBound(column_index, num_columns));
- }
-
- // We don't check iterators here because iterator may be infinite
- Ok(Self {
- column_index,
- row_group_indices,
- file_reader,
- })
- }
-}
-
-impl Iterator for FilePageIterator {
- type Item = Result<Box<dyn PageReader>>;
-
- fn next(&mut self) -> Option<Result<Box<dyn PageReader>>> {
- self.row_group_indices.next().map(|row_group_index| {
- self.file_reader
- .get_row_group(row_group_index)
- .and_then(|r| r.get_column_page_reader(self.column_index))
- })
- }
-}
-
-impl PageIterator for FilePageIterator {
- fn schema(&mut self) -> Result<SchemaDescPtr> {
- Ok(self
- .file_reader
- .metadata()
- .file_metadata()
- .schema_descr_ptr())
- }
-
- fn column_schema(&mut self) -> Result<ColumnDescPtr> {
- self.schema().map(|s| s.column(self.column_index))
- }
-}
diff --git a/parquet/src/file/serialized_reader.rs b/parquet/src/file/serialized_reader.rs
deleted file mode 100644
index a4d79a3..0000000
--- a/parquet/src/file/serialized_reader.rs
+++ /dev/null
@@ -1,779 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-//! Contains implementations of the reader traits FileReader, RowGroupReader and PageReader
-//! Also contains implementations of the ChunkReader for files (with buffering) and byte arrays (RAM)
-
-use std::{convert::TryFrom, fs::File, io::Read, path::Path, sync::Arc};
-
-use parquet_format::{PageHeader, PageType};
-use thrift::protocol::TCompactInputProtocol;
-
-use crate::basic::{Compression, Encoding, Type};
-use crate::column::page::{Page, PageReader};
-use crate::compression::{create_codec, Codec};
-use crate::errors::{ParquetError, Result};
-use crate::file::{footer, metadata::*, reader::*, statistics};
-use crate::record::reader::RowIter;
-use crate::record::Row;
-use crate::schema::types::Type as SchemaType;
-use crate::util::{io::TryClone, memory::ByteBufferPtr};
-
-// export `SliceableCursor` and `FileSource` publically so clients can
-// re-use the logic in their own ParquetFileWriter wrappers
-pub use crate::util::{cursor::SliceableCursor, io::FileSource};
-
-// ----------------------------------------------------------------------
-// Implementations of traits facilitating the creation of a new reader
-
-impl Length for File {
- fn len(&self) -> u64 {
- self.metadata().map(|m| m.len()).unwrap_or(0u64)
- }
-}
-
-impl TryClone for File {
- fn try_clone(&self) -> std::io::Result<Self> {
- self.try_clone()
- }
-}
-
-impl ChunkReader for File {
- type T = FileSource<File>;
-
- fn get_read(&self, start: u64, length: usize) -> Result<Self::T> {
- Ok(FileSource::new(self, start, length))
- }
-}
-
-impl Length for SliceableCursor {
- fn len(&self) -> u64 {
- SliceableCursor::len(self)
- }
-}
-
-impl ChunkReader for SliceableCursor {
- type T = SliceableCursor;
-
- fn get_read(&self, start: u64, length: usize) -> Result<Self::T> {
- self.slice(start, length).map_err(|e| e.into())
- }
-}
-
-impl TryFrom<File> for SerializedFileReader<File> {
- type Error = ParquetError;
-
- fn try_from(file: File) -> Result<Self> {
- Self::new(file)
- }
-}
-
-impl<'a> TryFrom<&'a Path> for SerializedFileReader<File> {
- type Error = ParquetError;
-
- fn try_from(path: &Path) -> Result<Self> {
- let file = File::open(path)?;
- Self::try_from(file)
- }
-}
-
-impl TryFrom<String> for SerializedFileReader<File> {
- type Error = ParquetError;
-
- fn try_from(path: String) -> Result<Self> {
- Self::try_from(Path::new(&path))
- }
-}
-
-impl<'a> TryFrom<&'a str> for SerializedFileReader<File> {
- type Error = ParquetError;
-
- fn try_from(path: &str) -> Result<Self> {
- Self::try_from(Path::new(&path))
- }
-}
-
-/// Conversion into a [`RowIter`](crate::record::reader::RowIter)
-/// using the full file schema over all row groups.
-impl IntoIterator for SerializedFileReader<File> {
- type Item = Row;
- type IntoIter = RowIter<'static>;
-
- fn into_iter(self) -> Self::IntoIter {
- RowIter::from_file_into(Box::new(self))
- }
-}
-
-// ----------------------------------------------------------------------
-// Implementations of file & row group readers
-
-/// A serialized implementation for Parquet [`FileReader`].
-pub struct SerializedFileReader<R: ChunkReader> {
- chunk_reader: Arc<R>,
- metadata: ParquetMetaData,
-}
-
-impl<R: 'static + ChunkReader> SerializedFileReader<R> {
- /// Creates file reader from a Parquet file.
- /// Returns error if Parquet file does not exist or is corrupt.
- pub fn new(chunk_reader: R) -> Result<Self> {
- let metadata = footer::parse_metadata(&chunk_reader)?;
- Ok(Self {
- chunk_reader: Arc::new(chunk_reader),
- metadata,
- })
- }
-
- /// Filters row group metadata to only those row groups,
- /// for which the predicate function returns true
- pub fn filter_row_groups(
- &mut self,
- predicate: &dyn Fn(&RowGroupMetaData, usize) -> bool,
- ) {
- let mut filtered_row_groups = Vec::<RowGroupMetaData>::new();
- for (i, row_group_metadata) in self.metadata.row_groups().iter().enumerate() {
- if predicate(row_group_metadata, i) {
- filtered_row_groups.push(row_group_metadata.clone());
- }
- }
- self.metadata = ParquetMetaData::new(
- self.metadata.file_metadata().clone(),
- filtered_row_groups,
- );
- }
-}
-
-impl<R: 'static + ChunkReader> FileReader for SerializedFileReader<R> {
- fn metadata(&self) -> &ParquetMetaData {
- &self.metadata
- }
-
- fn num_row_groups(&self) -> usize {
- self.metadata.num_row_groups()
- }
-
- fn get_row_group(&self, i: usize) -> Result<Box<dyn RowGroupReader + '_>> {
- let row_group_metadata = self.metadata.row_group(i);
- // Row groups should be processed sequentially.
- let f = Arc::clone(&self.chunk_reader);
- Ok(Box::new(SerializedRowGroupReader::new(
- f,
- row_group_metadata,
- )))
- }
-
- fn get_row_iter(&self, projection: Option<SchemaType>) -> Result<RowIter> {
- RowIter::from_file(projection, self)
- }
-}
-
-/// A serialized implementation for Parquet [`RowGroupReader`].
-pub struct SerializedRowGroupReader<'a, R: ChunkReader> {
- chunk_reader: Arc<R>,
- metadata: &'a RowGroupMetaData,
-}
-
-impl<'a, R: ChunkReader> SerializedRowGroupReader<'a, R> {
- /// Creates new row group reader from a file and row group metadata.
- fn new(chunk_reader: Arc<R>, metadata: &'a RowGroupMetaData) -> Self {
- Self {
- chunk_reader,
- metadata,
- }
- }
-}
-
-impl<'a, R: 'static + ChunkReader> RowGroupReader for SerializedRowGroupReader<'a, R> {
- fn metadata(&self) -> &RowGroupMetaData {
- &self.metadata
- }
-
- fn num_columns(&self) -> usize {
- self.metadata.num_columns()
- }
-
- // TODO: fix PARQUET-816
- fn get_column_page_reader(&self, i: usize) -> Result<Box<dyn PageReader>> {
- let col = self.metadata.column(i);
- let (col_start, col_length) = col.byte_range();
- let file_chunk = self.chunk_reader.get_read(col_start, col_length as usize)?;
- let page_reader = SerializedPageReader::new(
- file_chunk,
- col.num_values(),
- col.compression(),
- col.column_descr().physical_type(),
- )?;
- Ok(Box::new(page_reader))
- }
-
- fn get_row_iter(&self, projection: Option<SchemaType>) -> Result<RowIter> {
- RowIter::from_row_group(projection, self)
- }
-}
-
-/// A serialized implementation for Parquet [`PageReader`].
-pub struct SerializedPageReader<T: Read> {
- // The file source buffer which references exactly the bytes for the column trunk
- // to be read by this page reader.
- buf: T,
-
- // The compression codec for this column chunk. Only set for non-PLAIN codec.
- decompressor: Option<Box<dyn Codec>>,
-
- // The number of values we have seen so far.
- seen_num_values: i64,
-
- // The number of total values in this column chunk.
- total_num_values: i64,
-
- // Column chunk type.
- physical_type: Type,
-}
-
-impl<T: Read> SerializedPageReader<T> {
- /// Creates a new serialized page reader from file source.
- pub fn new(
- buf: T,
- total_num_values: i64,
- compression: Compression,
- physical_type: Type,
- ) -> Result<Self> {
- let decompressor = create_codec(compression)?;
- let result = Self {
- buf,
- total_num_values,
- seen_num_values: 0,
- decompressor,
- physical_type,
- };
- Ok(result)
- }
-
- /// Reads Page header from Thrift.
- fn read_page_header(&mut self) -> Result<PageHeader> {
- let mut prot = TCompactInputProtocol::new(&mut self.buf);
- let page_header = PageHeader::read_from_in_protocol(&mut prot)?;
- Ok(page_header)
- }
-}
-
-impl<T: Read> Iterator for SerializedPageReader<T> {
- type Item = Result<Page>;
-
- fn next(&mut self) -> Option<Self::Item> {
- self.get_next_page().transpose()
- }
-}
-
-impl<T: Read> PageReader for SerializedPageReader<T> {
- fn get_next_page(&mut self) -> Result<Option<Page>> {
- while self.seen_num_values < self.total_num_values {
- let page_header = self.read_page_header()?;
-
- // When processing data page v2, depending on enabled compression for the
- // page, we should account for uncompressed data ('offset') of
- // repetition and definition levels.
- //
- // We always use 0 offset for other pages other than v2, `true` flag means
- // that compression will be applied if decompressor is defined
- let mut offset: usize = 0;
- let mut can_decompress = true;
-
- if let Some(ref header_v2) = page_header.data_page_header_v2 {
- offset = (header_v2.definition_levels_byte_length
- + header_v2.repetition_levels_byte_length)
- as usize;
- // When is_compressed flag is missing the page is considered compressed
- can_decompress = header_v2.is_compressed.unwrap_or(true);
- }
-
- let compressed_len = page_header.compressed_page_size as usize - offset;
- let uncompressed_len = page_header.uncompressed_page_size as usize - offset;
- // We still need to read all bytes from buffered stream
- let mut buffer = vec![0; offset + compressed_len];
- self.buf.read_exact(&mut buffer)?;
-
- // TODO: page header could be huge because of statistics. We should set a
- // maximum page header size and abort if that is exceeded.
- if let Some(decompressor) = self.decompressor.as_mut() {
- if can_decompress {
- let mut decompressed_buffer = Vec::with_capacity(uncompressed_len);
- let decompressed_size = decompressor
- .decompress(&buffer[offset..], &mut decompressed_buffer)?;
- if decompressed_size != uncompressed_len {
- return Err(general_err!(
- "Actual decompressed size doesn't match the expected one ({} vs {})",
- decompressed_size,
- uncompressed_len
- ));
- }
- if offset == 0 {
- buffer = decompressed_buffer;
- } else {
- // Prepend saved offsets to the buffer
- buffer.truncate(offset);
- buffer.append(&mut decompressed_buffer);
- }
- }
- }
-
- let result = match page_header.type_ {
- PageType::DictionaryPage => {
- assert!(page_header.dictionary_page_header.is_some());
- let dict_header =
- page_header.dictionary_page_header.as_ref().unwrap();
- let is_sorted = dict_header.is_sorted.unwrap_or(false);
- Page::DictionaryPage {
- buf: ByteBufferPtr::new(buffer),
- num_values: dict_header.num_values as u32,
- encoding: Encoding::from(dict_header.encoding),
- is_sorted,
- }
- }
- PageType::DataPage => {
- assert!(page_header.data_page_header.is_some());
- let header = page_header.data_page_header.unwrap();
- self.seen_num_values += header.num_values as i64;
- Page::DataPage {
- buf: ByteBufferPtr::new(buffer),
- num_values: header.num_values as u32,
- encoding: Encoding::from(header.encoding),
- def_level_encoding: Encoding::from(
- header.definition_level_encoding,
- ),
- rep_level_encoding: Encoding::from(
- header.repetition_level_encoding,
- ),
- statistics: statistics::from_thrift(
- self.physical_type,
- header.statistics,
- ),
- }
- }
- PageType::DataPageV2 => {
- assert!(page_header.data_page_header_v2.is_some());
- let header = page_header.data_page_header_v2.unwrap();
- let is_compressed = header.is_compressed.unwrap_or(true);
- self.seen_num_values += header.num_values as i64;
- Page::DataPageV2 {
- buf: ByteBufferPtr::new(buffer),
- num_values: header.num_values as u32,
- encoding: Encoding::from(header.encoding),
- num_nulls: header.num_nulls as u32,
- num_rows: header.num_rows as u32,
- def_levels_byte_len: header.definition_levels_byte_length as u32,
- rep_levels_byte_len: header.repetition_levels_byte_length as u32,
- is_compressed,
- statistics: statistics::from_thrift(
- self.physical_type,
- header.statistics,
- ),
- }
- }
- _ => {
- // For unknown page type (e.g., INDEX_PAGE), skip and read next.
- continue;
- }
- };
- return Ok(Some(result));
- }
-
- // We are at the end of this column chunk and no more page left. Return None.
- Ok(None)
- }
-}
-
-#[cfg(test)]
-mod tests {
- use super::*;
- use crate::basic::ColumnOrder;
- use crate::record::RowAccessor;
- use crate::schema::parser::parse_message_type;
- use crate::util::test_common::{get_test_file, get_test_path};
- use std::sync::Arc;
-
- #[test]
- fn test_cursor_and_file_has_the_same_behaviour() {
- let mut buf: Vec<u8> = Vec::new();
- get_test_file("alltypes_plain.parquet")
- .read_to_end(&mut buf)
- .unwrap();
- let cursor = SliceableCursor::new(buf);
- let read_from_cursor = SerializedFileReader::new(cursor).unwrap();
-
- let test_file = get_test_file("alltypes_plain.parquet");
- let read_from_file = SerializedFileReader::new(test_file).unwrap();
-
- let file_iter = read_from_file.get_row_iter(None).unwrap();
- let cursor_iter = read_from_cursor.get_row_iter(None).unwrap();
-
- assert!(file_iter.eq(cursor_iter));
- }
-
- #[test]
- fn test_file_reader_try_from() {
- // Valid file path
- let test_file = get_test_file("alltypes_plain.parquet");
- let test_path_buf = get_test_path("alltypes_plain.parquet");
- let test_path = test_path_buf.as_path();
- let test_path_str = test_path.to_str().unwrap();
-
- let reader = SerializedFileReader::try_from(test_file);
- assert!(reader.is_ok());
-
- let reader = SerializedFileReader::try_from(test_path);
- assert!(reader.is_ok());
-
- let reader = SerializedFileReader::try_from(test_path_str);
- assert!(reader.is_ok());
-
- let reader = SerializedFileReader::try_from(test_path_str.to_string());
- assert!(reader.is_ok());
-
- // Invalid file path
- let test_path = Path::new("invalid.parquet");
- let test_path_str = test_path.to_str().unwrap();
-
- let reader = SerializedFileReader::try_from(test_path);
- assert!(reader.is_err());
-
- let reader = SerializedFileReader::try_from(test_path_str);
- assert!(reader.is_err());
-
- let reader = SerializedFileReader::try_from(test_path_str.to_string());
- assert!(reader.is_err());
- }
-
- #[test]
- fn test_file_reader_into_iter() {
- let path = get_test_path("alltypes_plain.parquet");
- let vec = vec![path.clone(), path]
- .iter()
- .map(|p| SerializedFileReader::try_from(p.as_path()).unwrap())
- .flat_map(|r| r.into_iter())
- .flat_map(|r| r.get_int(0))
- .collect::<Vec<_>>();
-
- // rows in the parquet file are not sorted by "id"
- // each file contains [id:4, id:5, id:6, id:7, id:2, id:3, id:0, id:1]
- assert_eq!(vec, vec![4, 5, 6, 7, 2, 3, 0, 1, 4, 5, 6, 7, 2, 3, 0, 1]);
- }
-
- #[test]
- fn test_file_reader_into_iter_project() {
- let path = get_test_path("alltypes_plain.parquet");
- let result = vec![path]
- .iter()
- .map(|p| SerializedFileReader::try_from(p.as_path()).unwrap())
- .flat_map(|r| {
- let schema = "message schema { OPTIONAL INT32 id; }";
- let proj = parse_message_type(&schema).ok();
-
- r.into_iter().project(proj).unwrap()
- })
- .map(|r| format!("{}", r))
- .collect::<Vec<_>>()
- .join(",");
-
- assert_eq!(
- result,
- "{id: 4},{id: 5},{id: 6},{id: 7},{id: 2},{id: 3},{id: 0},{id: 1}"
- );
- }
-
- #[test]
- fn test_reuse_file_chunk() {
- // This test covers the case of maintaining the correct start position in a file
- // stream for each column reader after initializing and moving to the next one
- // (without necessarily reading the entire column).
- let test_file = get_test_file("alltypes_plain.parquet");
- let reader = SerializedFileReader::new(test_file).unwrap();
- let row_group = reader.get_row_group(0).unwrap();
-
- let mut page_readers = Vec::new();
- for i in 0..row_group.num_columns() {
- page_readers.push(row_group.get_column_page_reader(i).unwrap());
- }
-
- // Now buffer each col reader, we do not expect any failures like:
- // General("underlying Thrift error: end of file")
- for mut page_reader in page_readers {
- assert!(page_reader.get_next_page().is_ok());
- }
- }
-
- #[test]
- fn test_file_reader() {
- let test_file = get_test_file("alltypes_plain.parquet");
- let reader_result = SerializedFileReader::new(test_file);
- assert!(reader_result.is_ok());
- let reader = reader_result.unwrap();
-
- // Test contents in Parquet metadata
- let metadata = reader.metadata();
- assert_eq!(metadata.num_row_groups(), 1);
-
- // Test contents in file metadata
- let file_metadata = metadata.file_metadata();
- assert!(file_metadata.created_by().is_some());
- assert_eq!(
- file_metadata.created_by().as_ref().unwrap(),
- "impala version 1.3.0-INTERNAL (build 8a48ddb1eff84592b3fc06bc6f51ec120e1fffc9)"
- );
- assert!(file_metadata.key_value_metadata().is_none());
- assert_eq!(file_metadata.num_rows(), 8);
- assert_eq!(file_metadata.version(), 1);
- assert_eq!(file_metadata.column_orders(), None);
-
- // Test contents in row group metadata
- let row_group_metadata = metadata.row_group(0);
- assert_eq!(row_group_metadata.num_columns(), 11);
- assert_eq!(row_group_metadata.num_rows(), 8);
- assert_eq!(row_group_metadata.total_byte_size(), 671);
- // Check each column order
- for i in 0..row_group_metadata.num_columns() {
- assert_eq!(file_metadata.column_order(i), ColumnOrder::UNDEFINED);
- }
-
- // Test row group reader
- let row_group_reader_result = reader.get_row_group(0);
- assert!(row_group_reader_result.is_ok());
- let row_group_reader: Box<dyn RowGroupReader> = row_group_reader_result.unwrap();
- assert_eq!(
- row_group_reader.num_columns(),
- row_group_metadata.num_columns()
- );
- assert_eq!(
- row_group_reader.metadata().total_byte_size(),
- row_group_metadata.total_byte_size()
- );
-
- // Test page readers
- // TODO: test for every column
- let page_reader_0_result = row_group_reader.get_column_page_reader(0);
- assert!(page_reader_0_result.is_ok());
- let mut page_reader_0: Box<dyn PageReader> = page_reader_0_result.unwrap();
- let mut page_count = 0;
- while let Ok(Some(page)) = page_reader_0.get_next_page() {
- let is_expected_page = match page {
- Page::DictionaryPage {
- buf,
- num_values,
- encoding,
- is_sorted,
- } => {
- assert_eq!(buf.len(), 32);
- assert_eq!(num_values, 8);
- assert_eq!(encoding, Encoding::PLAIN_DICTIONARY);
- assert_eq!(is_sorted, false);
- true
- }
- Page::DataPage {
- buf,
- num_values,
- encoding,
- def_level_encoding,
- rep_level_encoding,
- statistics,
- } => {
- assert_eq!(buf.len(), 11);
- assert_eq!(num_values, 8);
- assert_eq!(encoding, Encoding::PLAIN_DICTIONARY);
- assert_eq!(def_level_encoding, Encoding::RLE);
- assert_eq!(rep_level_encoding, Encoding::BIT_PACKED);
- assert!(statistics.is_none());
- true
- }
- _ => false,
- };
- assert!(is_expected_page);
- page_count += 1;
- }
- assert_eq!(page_count, 2);
- }
-
- #[test]
- fn test_file_reader_datapage_v2() {
- let test_file = get_test_file("datapage_v2.snappy.parquet");
- let reader_result = SerializedFileReader::new(test_file);
- assert!(reader_result.is_ok());
- let reader = reader_result.unwrap();
-
- // Test contents in Parquet metadata
- let metadata = reader.metadata();
- assert_eq!(metadata.num_row_groups(), 1);
-
- // Test contents in file metadata
- let file_metadata = metadata.file_metadata();
- assert!(file_metadata.created_by().is_some());
- assert_eq!(
- file_metadata.created_by().as_ref().unwrap(),
- "parquet-mr version 1.8.1 (build 4aba4dae7bb0d4edbcf7923ae1339f28fd3f7fcf)"
- );
- assert!(file_metadata.key_value_metadata().is_some());
- assert_eq!(
- file_metadata.key_value_metadata().to_owned().unwrap().len(),
- 1
- );
-
- assert_eq!(file_metadata.num_rows(), 5);
- assert_eq!(file_metadata.version(), 1);
- assert_eq!(file_metadata.column_orders(), None);
-
- let row_group_metadata = metadata.row_group(0);
-
- // Check each column order
- for i in 0..row_group_metadata.num_columns() {
- assert_eq!(file_metadata.column_order(i), ColumnOrder::UNDEFINED);
- }
-
- // Test row group reader
- let row_group_reader_result = reader.get_row_group(0);
- assert!(row_group_reader_result.is_ok());
- let row_group_reader: Box<dyn RowGroupReader> = row_group_reader_result.unwrap();
- assert_eq!(
- row_group_reader.num_columns(),
- row_group_metadata.num_columns()
- );
- assert_eq!(
- row_group_reader.metadata().total_byte_size(),
- row_group_metadata.total_byte_size()
- );
-
- // Test page readers
- // TODO: test for every column
- let page_reader_0_result = row_group_reader.get_column_page_reader(0);
- assert!(page_reader_0_result.is_ok());
- let mut page_reader_0: Box<dyn PageReader> = page_reader_0_result.unwrap();
- let mut page_count = 0;
- while let Ok(Some(page)) = page_reader_0.get_next_page() {
- let is_expected_page = match page {
- Page::DictionaryPage {
- buf,
- num_values,
- encoding,
- is_sorted,
- } => {
- assert_eq!(buf.len(), 7);
- assert_eq!(num_values, 1);
- assert_eq!(encoding, Encoding::PLAIN);
- assert_eq!(is_sorted, false);
- true
- }
- Page::DataPageV2 {
- buf,
- num_values,
- encoding,
- num_nulls,
- num_rows,
- def_levels_byte_len,
- rep_levels_byte_len,
- is_compressed,
- statistics,
- } => {
- assert_eq!(buf.len(), 4);
- assert_eq!(num_values, 5);
- assert_eq!(encoding, Encoding::RLE_DICTIONARY);
- assert_eq!(num_nulls, 1);
- assert_eq!(num_rows, 5);
- assert_eq!(def_levels_byte_len, 2);
- assert_eq!(rep_levels_byte_len, 0);
- assert_eq!(is_compressed, true);
- assert!(statistics.is_some());
- true
- }
- _ => false,
- };
- assert!(is_expected_page);
- page_count += 1;
- }
- assert_eq!(page_count, 2);
- }
-
- #[test]
- fn test_page_iterator() {
- let file = get_test_file("alltypes_plain.parquet");
- let file_reader = Arc::new(SerializedFileReader::new(file).unwrap());
-
- let mut page_iterator = FilePageIterator::new(0, file_reader.clone()).unwrap();
-
- // read first page
- let page = page_iterator.next();
- assert!(page.is_some());
- assert!(page.unwrap().is_ok());
-
- // reach end of file
- let page = page_iterator.next();
- assert!(page.is_none());
-
- let row_group_indices = Box::new(0..1);
- let mut page_iterator =
- FilePageIterator::with_row_groups(0, row_group_indices, file_reader).unwrap();
-
- // read first page
- let page = page_iterator.next();
- assert!(page.is_some());
- assert!(page.unwrap().is_ok());
-
- // reach end of file
- let page = page_iterator.next();
- assert!(page.is_none());
- }
-
- #[test]
- fn test_file_reader_key_value_metadata() {
- let file = get_test_file("binary.parquet");
- let file_reader = Arc::new(SerializedFileReader::new(file).unwrap());
-
- let metadata = file_reader
- .metadata
- .file_metadata()
- .key_value_metadata()
- .as_ref()
- .unwrap();
-
- assert_eq!(metadata.len(), 3);
-
- assert_eq!(metadata.get(0).unwrap().key, "parquet.proto.descriptor");
-
- assert_eq!(metadata.get(1).unwrap().key, "writer.model.name");
- assert_eq!(metadata.get(1).unwrap().value, Some("protobuf".to_owned()));
-
- assert_eq!(metadata.get(2).unwrap().key, "parquet.proto.class");
- assert_eq!(
- metadata.get(2).unwrap().value,
- Some("foo.baz.Foobaz$Event".to_owned())
- );
- }
-
- #[test]
- fn test_file_reader_filter_row_groups() -> Result<()> {
- let test_file = get_test_file("alltypes_plain.parquet");
- let mut reader = SerializedFileReader::new(test_file)?;
-
- // test initial number of row groups
- let metadata = reader.metadata();
- assert_eq!(metadata.num_row_groups(), 1);
-
- // test filtering out all row groups
- reader.filter_row_groups(&|_, _| false);
- let metadata = reader.metadata();
- assert_eq!(metadata.num_row_groups(), 0);
-
- Ok(())
- }
-}
diff --git a/parquet/src/file/statistics.rs b/parquet/src/file/statistics.rs
deleted file mode 100644
index 4f5d0e9..0000000
--- a/parquet/src/file/statistics.rs
+++ /dev/null
@@ -1,664 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-//! Contains definitions for working with Parquet statistics.
-//!
-//! Though some common methods are available on enum, use pattern match to extract
-//! actual min and max values from statistics, see below:
-//!
-//! ```rust
-//! use parquet::file::statistics::Statistics;
-//!
-//! let stats = Statistics::int32(Some(1), Some(10), None, 3, true);
-//! assert_eq!(stats.null_count(), 3);
-//! assert!(stats.has_min_max_set());
-//! assert!(stats.is_min_max_deprecated());
-//!
-//! match stats {
-//! Statistics::Int32(ref typed) => {
-//! assert_eq!(*typed.min(), 1);
-//! assert_eq!(*typed.max(), 10);
-//! }
-//! _ => {}
-//! }
-//! ```
-
-use std::{cmp, fmt};
-
-use byteorder::{ByteOrder, LittleEndian};
-use parquet_format::Statistics as TStatistics;
-
-use crate::basic::Type;
-use crate::data_type::*;
-use crate::util::bit_util::from_ne_slice;
-
-// Macro to generate methods create Statistics.
-macro_rules! statistics_new_func {
- ($func:ident, $vtype:ty, $stat:ident) => {
- pub fn $func(
- min: $vtype,
- max: $vtype,
- distinct: Option<u64>,
- nulls: u64,
- is_deprecated: bool,
- ) -> Self {
- Statistics::$stat(TypedStatistics::new(
- min,
- max,
- distinct,
- nulls,
- is_deprecated,
- ))
- }
- };
-}
-
-// Macro to generate getter functions for Statistics.
-macro_rules! statistics_enum_func {
- ($self:ident, $func:ident) => {{
- match *$self {
- Statistics::Boolean(ref typed) => typed.$func(),
- Statistics::Int32(ref typed) => typed.$func(),
- Statistics::Int64(ref typed) => typed.$func(),
- Statistics::Int96(ref typed) => typed.$func(),
- Statistics::Float(ref typed) => typed.$func(),
- Statistics::Double(ref typed) => typed.$func(),
- Statistics::ByteArray(ref typed) => typed.$func(),
- Statistics::FixedLenByteArray(ref typed) => typed.$func(),
- }
- }};
-}
-
-/// Converts Thrift definition into `Statistics`.
-pub fn from_thrift(
- physical_type: Type,
- thrift_stats: Option<TStatistics>,
-) -> Option<Statistics> {
- match thrift_stats {
- Some(stats) => {
- // Number of nulls recorded, when it is not available, we just mark it as 0.
- let null_count = stats.null_count.unwrap_or(0);
- assert!(
- null_count >= 0,
- "Statistics null count is negative ({})",
- null_count
- );
-
- // Generic null count.
- let null_count = null_count as u64;
- // Generic distinct count (count of distinct values occurring)
- let distinct_count = stats.distinct_count.map(|value| value as u64);
- // Whether or not statistics use deprecated min/max fields.
- let old_format = stats.min_value.is_none() && stats.max_value.is_none();
- // Generic min value as bytes.
- let min = if old_format {
- stats.min
- } else {
- stats.min_value
- };
- // Generic max value as bytes.
- let max = if old_format {
- stats.max
- } else {
- stats.max_value
- };
-
- // Values are encoded using PLAIN encoding definition, except that
- // variable-length byte arrays do not include a length prefix.
- //
- // Instead of using actual decoder, we manually convert values.
- let res = match physical_type {
- Type::BOOLEAN => Statistics::boolean(
- min.map(|data| data[0] != 0),
- max.map(|data| data[0] != 0),
- distinct_count,
- null_count,
- old_format,
- ),
- Type::INT32 => Statistics::int32(
- min.map(|data| LittleEndian::read_i32(&data)),
- max.map(|data| LittleEndian::read_i32(&data)),
- distinct_count,
- null_count,
- old_format,
- ),
- Type::INT64 => Statistics::int64(
- min.map(|data| LittleEndian::read_i64(&data)),
- max.map(|data| LittleEndian::read_i64(&data)),
- distinct_count,
- null_count,
- old_format,
- ),
- Type::INT96 => {
- // INT96 statistics may not be correct, because comparison is signed
- // byte-wise, not actual timestamps. It is recommended to ignore
- // min/max statistics for INT96 columns.
- let min = min.map(|data| {
- assert_eq!(data.len(), 12);
- from_ne_slice::<Int96>(&data)
- });
- let max = max.map(|data| {
- assert_eq!(data.len(), 12);
- from_ne_slice::<Int96>(&data)
- });
- Statistics::int96(min, max, distinct_count, null_count, old_format)
- }
- Type::FLOAT => Statistics::float(
- min.map(|data| LittleEndian::read_f32(&data)),
- max.map(|data| LittleEndian::read_f32(&data)),
- distinct_count,
- null_count,
- old_format,
- ),
- Type::DOUBLE => Statistics::double(
- min.map(|data| LittleEndian::read_f64(&data)),
- max.map(|data| LittleEndian::read_f64(&data)),
- distinct_count,
- null_count,
- old_format,
- ),
- Type::BYTE_ARRAY => Statistics::byte_array(
- min.map(ByteArray::from),
- max.map(ByteArray::from),
- distinct_count,
- null_count,
- old_format,
- ),
- Type::FIXED_LEN_BYTE_ARRAY => Statistics::fixed_len_byte_array(
- min.map(ByteArray::from).map(FixedLenByteArray::from),
- max.map(ByteArray::from).map(FixedLenByteArray::from),
- distinct_count,
- null_count,
- old_format,
- ),
- };
-
- Some(res)
- }
- None => None,
- }
-}
-
-// Convert Statistics into Thrift definition.
-pub fn to_thrift(stats: Option<&Statistics>) -> Option<TStatistics> {
- let stats = stats?;
-
- let mut thrift_stats = TStatistics {
- max: None,
- min: None,
- null_count: if stats.has_nulls() {
- Some(stats.null_count() as i64)
- } else {
- None
- },
- distinct_count: stats.distinct_count().map(|value| value as i64),
- max_value: None,
- min_value: None,
- };
-
- // Get min/max if set.
- let (min, max) = if stats.has_min_max_set() {
- (
- Some(stats.min_bytes().to_vec()),
- Some(stats.max_bytes().to_vec()),
- )
- } else {
- (None, None)
- };
-
- if stats.is_min_max_deprecated() {
- thrift_stats.min = min;
- thrift_stats.max = max;
- } else {
- thrift_stats.min_value = min;
- thrift_stats.max_value = max;
- }
-
- Some(thrift_stats)
-}
-
-/// Statistics for a column chunk and data page.
-#[derive(Debug, Clone, PartialEq)]
-pub enum Statistics {
- Boolean(TypedStatistics<BoolType>),
- Int32(TypedStatistics<Int32Type>),
- Int64(TypedStatistics<Int64Type>),
- Int96(TypedStatistics<Int96Type>),
- Float(TypedStatistics<FloatType>),
- Double(TypedStatistics<DoubleType>),
- ByteArray(TypedStatistics<ByteArrayType>),
- FixedLenByteArray(TypedStatistics<FixedLenByteArrayType>),
-}
-
-impl Statistics {
- statistics_new_func![boolean, Option<bool>, Boolean];
-
- statistics_new_func![int32, Option<i32>, Int32];
-
- statistics_new_func![int64, Option<i64>, Int64];
-
- statistics_new_func![int96, Option<Int96>, Int96];
-
- statistics_new_func![float, Option<f32>, Float];
-
- statistics_new_func![double, Option<f64>, Double];
-
- statistics_new_func![byte_array, Option<ByteArray>, ByteArray];
-
- statistics_new_func![
- fixed_len_byte_array,
- Option<FixedLenByteArray>,
- FixedLenByteArray
- ];
-
- /// Returns `true` if statistics have old `min` and `max` fields set.
- /// This means that the column order is likely to be undefined, which, for old files
- /// could mean a signed sort order of values.
- ///
- /// Refer to [`ColumnOrder`](crate::basic::ColumnOrder) and
- /// [`SortOrder`](crate::basic::SortOrder) for more information.
- pub fn is_min_max_deprecated(&self) -> bool {
- statistics_enum_func![self, is_min_max_deprecated]
- }
-
- /// Returns optional value of number of distinct values occurring.
- /// When it is `None`, the value should be ignored.
- pub fn distinct_count(&self) -> Option<u64> {
- statistics_enum_func![self, distinct_count]
- }
-
- /// Returns number of null values for the column.
- /// Note that this includes all nulls when column is part of the complex type.
- pub fn null_count(&self) -> u64 {
- statistics_enum_func![self, null_count]
- }
-
- /// Returns `true` if statistics collected any null values, `false` otherwise.
- pub fn has_nulls(&self) -> bool {
- self.null_count() > 0
- }
-
- /// Returns `true` if min value and max value are set.
- /// Normally both min/max values will be set to `Some(value)` or `None`.
- pub fn has_min_max_set(&self) -> bool {
- statistics_enum_func![self, has_min_max_set]
- }
-
- /// Returns slice of bytes that represent min value.
- /// Panics if min value is not set.
- pub fn min_bytes(&self) -> &[u8] {
- statistics_enum_func![self, min_bytes]
- }
-
- /// Returns slice of bytes that represent max value.
- /// Panics if max value is not set.
- pub fn max_bytes(&self) -> &[u8] {
- statistics_enum_func![self, max_bytes]
- }
-
- /// Returns physical type associated with statistics.
- pub fn physical_type(&self) -> Type {
- match self {
- Statistics::Boolean(_) => Type::BOOLEAN,
- Statistics::Int32(_) => Type::INT32,
- Statistics::Int64(_) => Type::INT64,
- Statistics::Int96(_) => Type::INT96,
- Statistics::Float(_) => Type::FLOAT,
- Statistics::Double(_) => Type::DOUBLE,
- Statistics::ByteArray(_) => Type::BYTE_ARRAY,
- Statistics::FixedLenByteArray(_) => Type::FIXED_LEN_BYTE_ARRAY,
- }
- }
-}
-
-impl fmt::Display for Statistics {
- fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
- match self {
- Statistics::Boolean(typed) => write!(f, "{}", typed),
- Statistics::Int32(typed) => write!(f, "{}", typed),
- Statistics::Int64(typed) => write!(f, "{}", typed),
- Statistics::Int96(typed) => write!(f, "{}", typed),
- Statistics::Float(typed) => write!(f, "{}", typed),
- Statistics::Double(typed) => write!(f, "{}", typed),
- Statistics::ByteArray(typed) => write!(f, "{}", typed),
- Statistics::FixedLenByteArray(typed) => write!(f, "{}", typed),
- }
- }
-}
-
-/// Typed implementation for [`Statistics`].
-#[derive(Clone)]
-pub struct TypedStatistics<T: DataType> {
- min: Option<T::T>,
- max: Option<T::T>,
- // Distinct count could be omitted in some cases
- distinct_count: Option<u64>,
- null_count: u64,
- is_min_max_deprecated: bool,
-}
-
-impl<T: DataType> TypedStatistics<T> {
- /// Creates new typed statistics.
- pub fn new(
- min: Option<T::T>,
- max: Option<T::T>,
- distinct_count: Option<u64>,
- null_count: u64,
- is_min_max_deprecated: bool,
- ) -> Self {
- Self {
- min,
- max,
- distinct_count,
- null_count,
- is_min_max_deprecated,
- }
- }
-
- /// Returns min value of the statistics.
- ///
- /// Panics if min value is not set, e.g. all values are `null`.
- /// Use `has_min_max_set` method to check that.
- pub fn min(&self) -> &T::T {
- self.min.as_ref().unwrap()
- }
-
- /// Returns max value of the statistics.
- ///
- /// Panics if max value is not set, e.g. all values are `null`.
- /// Use `has_min_max_set` method to check that.
- pub fn max(&self) -> &T::T {
- self.max.as_ref().unwrap()
- }
-
- /// Returns min value as bytes of the statistics.
- ///
- /// Panics if min value is not set, use `has_min_max_set` method to check
- /// if values are set.
- pub fn min_bytes(&self) -> &[u8] {
- self.min().as_bytes()
- }
-
- /// Returns max value as bytes of the statistics.
- ///
- /// Panics if max value is not set, use `has_min_max_set` method to check
- /// if values are set.
- pub fn max_bytes(&self) -> &[u8] {
- self.max().as_bytes()
- }
-
- /// Whether or not min and max values are set.
- /// Normally both min/max values will be set to `Some(value)` or `None`.
- fn has_min_max_set(&self) -> bool {
- self.min.is_some() && self.max.is_some()
- }
-
- /// Returns optional value of number of distinct values occurring.
- fn distinct_count(&self) -> Option<u64> {
- self.distinct_count
- }
-
- /// Returns null count.
- fn null_count(&self) -> u64 {
- self.null_count
- }
-
- /// Returns `true` if statistics were created using old min/max fields.
- fn is_min_max_deprecated(&self) -> bool {
- self.is_min_max_deprecated
- }
-}
-
-impl<T: DataType> fmt::Display for TypedStatistics<T> {
- fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
- write!(f, "{{")?;
- write!(f, "min: ")?;
- match self.min {
- Some(ref value) => write!(f, "{}", value)?,
- None => write!(f, "N/A")?,
- }
- write!(f, ", max: ")?;
- match self.max {
- Some(ref value) => write!(f, "{}", value)?,
- None => write!(f, "N/A")?,
- }
- write!(f, ", distinct_count: ")?;
- match self.distinct_count {
- Some(value) => write!(f, "{}", value)?,
- None => write!(f, "N/A")?,
- }
- write!(f, ", null_count: {}", self.null_count)?;
- write!(f, ", min_max_deprecated: {}", self.is_min_max_deprecated)?;
- write!(f, "}}")
- }
-}
-
-impl<T: DataType> fmt::Debug for TypedStatistics<T> {
- fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
- write!(
- f,
- "{{min: {:?}, max: {:?}, distinct_count: {:?}, null_count: {}, \
- min_max_deprecated: {}}}",
- self.min,
- self.max,
- self.distinct_count,
- self.null_count,
- self.is_min_max_deprecated
- )
- }
-}
-
-impl<T: DataType> cmp::PartialEq for TypedStatistics<T> {
- fn eq(&self, other: &TypedStatistics<T>) -> bool {
- self.min == other.min
- && self.max == other.max
- && self.distinct_count == other.distinct_count
- && self.null_count == other.null_count
- && self.is_min_max_deprecated == other.is_min_max_deprecated
- }
-}
-
-#[cfg(test)]
-mod tests {
- use super::*;
-
- #[test]
- fn test_statistics_min_max_bytes() {
- let stats = Statistics::int32(Some(-123), Some(234), None, 1, false);
- assert!(stats.has_min_max_set());
- assert_eq!(stats.min_bytes(), (-123).as_bytes());
- assert_eq!(stats.max_bytes(), 234.as_bytes());
-
- let stats = Statistics::byte_array(
- Some(ByteArray::from(vec![1, 2, 3])),
- Some(ByteArray::from(vec![3, 4, 5])),
- None,
- 1,
- true,
- );
- assert!(stats.has_min_max_set());
- assert_eq!(stats.min_bytes(), &[1, 2, 3]);
- assert_eq!(stats.max_bytes(), &[3, 4, 5]);
- }
-
- #[test]
- #[should_panic(expected = "Statistics null count is negative (-10)")]
- fn test_statistics_negative_null_count() {
- let thrift_stats = TStatistics {
- max: None,
- min: None,
- null_count: Some(-10),
- distinct_count: None,
- max_value: None,
- min_value: None,
- };
-
- from_thrift(Type::INT32, Some(thrift_stats));
- }
-
- #[test]
- fn test_statistics_thrift_none() {
- assert_eq!(from_thrift(Type::INT32, None), None);
- assert_eq!(from_thrift(Type::BYTE_ARRAY, None), None);
- }
-
- #[test]
- fn test_statistics_debug() {
- let stats = Statistics::int32(Some(1), Some(12), None, 12, true);
- assert_eq!(
- format!("{:?}", stats),
- "Int32({min: Some(1), max: Some(12), distinct_count: None, null_count: 12, \
- min_max_deprecated: true})"
- );
-
- let stats = Statistics::int32(None, None, None, 7, false);
- assert_eq!(
- format!("{:?}", stats),
- "Int32({min: None, max: None, distinct_count: None, null_count: 7, \
- min_max_deprecated: false})"
- )
- }
-
- #[test]
- fn test_statistics_display() {
- let stats = Statistics::int32(Some(1), Some(12), None, 12, true);
- assert_eq!(
- format!("{}", stats),
- "{min: 1, max: 12, distinct_count: N/A, null_count: 12, min_max_deprecated: true}"
- );
-
- let stats = Statistics::int64(None, None, None, 7, false);
- assert_eq!(
- format!("{}", stats),
- "{min: N/A, max: N/A, distinct_count: N/A, null_count: 7, min_max_deprecated: \
- false}"
- );
-
- let stats = Statistics::int96(
- Some(Int96::from(vec![1, 0, 0])),
- Some(Int96::from(vec![2, 3, 4])),
- None,
- 3,
- true,
- );
- assert_eq!(
- format!("{}", stats),
- "{min: [1, 0, 0], max: [2, 3, 4], distinct_count: N/A, null_count: 3, \
- min_max_deprecated: true}"
- );
-
- let stats = Statistics::byte_array(
- Some(ByteArray::from(vec![1u8])),
- Some(ByteArray::from(vec![2u8])),
- Some(5),
- 7,
- false,
- );
- assert_eq!(
- format!("{}", stats),
- "{min: [1], max: [2], distinct_count: 5, null_count: 7, min_max_deprecated: false}"
- );
- }
-
- #[test]
- fn test_statistics_partial_eq() {
- let expected = Statistics::int32(Some(12), Some(45), None, 11, true);
-
- assert!(Statistics::int32(Some(12), Some(45), None, 11, true) == expected);
- assert!(Statistics::int32(Some(11), Some(45), None, 11, true) != expected);
- assert!(Statistics::int32(Some(12), Some(44), None, 11, true) != expected);
- assert!(Statistics::int32(Some(12), Some(45), None, 23, true) != expected);
- assert!(Statistics::int32(Some(12), Some(45), None, 11, false) != expected);
-
- assert!(
- Statistics::int32(Some(12), Some(45), None, 11, false)
- != Statistics::int64(Some(12), Some(45), None, 11, false)
- );
-
- assert!(
- Statistics::boolean(Some(false), Some(true), None, 0, true)
- != Statistics::double(Some(1.2), Some(4.5), None, 0, true)
- );
-
- assert!(
- Statistics::byte_array(
- Some(ByteArray::from(vec![1, 2, 3])),
- Some(ByteArray::from(vec![1, 2, 3])),
- None,
- 0,
- true
- ) != Statistics::fixed_len_byte_array(
- Some(ByteArray::from(vec![1, 2, 3]).into()),
- Some(ByteArray::from(vec![1, 2, 3]).into()),
- None,
- 0,
- true
- )
- );
- }
-
- #[test]
- fn test_statistics_from_thrift() {
- // Helper method to check statistics conversion.
- fn check_stats(stats: Statistics) {
- let tpe = stats.physical_type();
- let thrift_stats = to_thrift(Some(&stats));
- assert_eq!(from_thrift(tpe, thrift_stats), Some(stats));
- }
-
- check_stats(Statistics::boolean(Some(false), Some(true), None, 7, true));
- check_stats(Statistics::boolean(Some(false), Some(true), None, 7, true));
- check_stats(Statistics::boolean(Some(false), Some(true), None, 0, false));
- check_stats(Statistics::boolean(Some(true), Some(true), None, 7, true));
- check_stats(Statistics::boolean(Some(false), Some(false), None, 7, true));
- check_stats(Statistics::boolean(None, None, None, 7, true));
-
- check_stats(Statistics::int32(Some(-100), Some(500), None, 7, true));
- check_stats(Statistics::int32(Some(-100), Some(500), None, 0, false));
- check_stats(Statistics::int32(None, None, None, 7, true));
-
- check_stats(Statistics::int64(Some(-100), Some(200), None, 7, true));
- check_stats(Statistics::int64(Some(-100), Some(200), None, 0, false));
- check_stats(Statistics::int64(None, None, None, 7, true));
-
- check_stats(Statistics::float(Some(1.2), Some(3.4), None, 7, true));
- check_stats(Statistics::float(Some(1.2), Some(3.4), None, 0, false));
- check_stats(Statistics::float(None, None, None, 7, true));
-
- check_stats(Statistics::double(Some(1.2), Some(3.4), None, 7, true));
- check_stats(Statistics::double(Some(1.2), Some(3.4), None, 0, false));
- check_stats(Statistics::double(None, None, None, 7, true));
-
- check_stats(Statistics::byte_array(
- Some(ByteArray::from(vec![1, 2, 3])),
- Some(ByteArray::from(vec![3, 4, 5])),
- None,
- 7,
- true,
- ));
- check_stats(Statistics::byte_array(None, None, None, 7, true));
-
- check_stats(Statistics::fixed_len_byte_array(
- Some(ByteArray::from(vec![1, 2, 3]).into()),
- Some(ByteArray::from(vec![3, 4, 5]).into()),
- None,
- 7,
- true,
- ));
- check_stats(Statistics::fixed_len_byte_array(None, None, None, 7, true));
- }
-}
diff --git a/parquet/src/file/writer.rs b/parquet/src/file/writer.rs
deleted file mode 100644
index e1c2dc6..0000000
--- a/parquet/src/file/writer.rs
+++ /dev/null
@@ -1,1164 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-//! Contains file writer API, and provides methods to write row groups and columns by
-//! using row group writers and column writers respectively.
-
-use std::{
- io::{Seek, SeekFrom, Write},
- sync::Arc,
-};
-
-use byteorder::{ByteOrder, LittleEndian};
-use parquet_format as parquet;
-use thrift::protocol::{TCompactOutputProtocol, TOutputProtocol};
-
-use crate::basic::PageType;
-use crate::column::{
- page::{CompressedPage, Page, PageWriteSpec, PageWriter},
- writer::{get_column_writer, ColumnWriter},
-};
-use crate::errors::{ParquetError, Result};
-use crate::file::{
- metadata::*, properties::WriterPropertiesPtr,
- statistics::to_thrift as statistics_to_thrift, FOOTER_SIZE, PARQUET_MAGIC,
-};
-use crate::schema::types::{self, SchemaDescPtr, SchemaDescriptor, TypePtr};
-use crate::util::io::{FileSink, Position};
-
-// Exposed publically so client code can implement [`ParquetWriter`]
-pub use crate::util::io::TryClone;
-
-// Exposed publically for convenience of writing Parquet to a buffer of bytes
-pub use crate::util::cursor::InMemoryWriteableCursor;
-
-// ----------------------------------------------------------------------
-// APIs for file & row group writers
-
-/// Parquet file writer API.
-/// Provides methods to write row groups sequentially.
-///
-/// The main workflow should be as following:
-/// - Create file writer, this will open a new file and potentially write some metadata.
-/// - Request a new row group writer by calling `next_row_group`.
-/// - Once finished writing row group, close row group writer by passing it into
-/// `close_row_group` method - this will finalise row group metadata and update metrics.
-/// - Write subsequent row groups, if necessary.
-/// - After all row groups have been written, close the file writer using `close` method.
-pub trait FileWriter {
- /// Creates new row group from this file writer.
- /// In case of IO error or Thrift error, returns `Err`.
- ///
- /// There is no limit on a number of row groups in a file; however, row groups have
- /// to be written sequentially. Every time the next row group is requested, the
- /// previous row group must be finalised and closed using `close_row_group` method.
- fn next_row_group(&mut self) -> Result<Box<dyn RowGroupWriter>>;
-
- /// Finalises and closes row group that was created using `next_row_group` method.
- /// After calling this method, the next row group is available for writes.
- fn close_row_group(
- &mut self,
- row_group_writer: Box<dyn RowGroupWriter>,
- ) -> Result<()>;
-
- /// Closes and finalises file writer, returning the file metadata.
- ///
- /// All row groups must be appended before this method is called.
- /// No writes are allowed after this point.
- ///
- /// Can be called multiple times. It is up to implementation to either result in
- /// no-op, or return an `Err` for subsequent calls.
- fn close(&mut self) -> Result<parquet::FileMetaData>;
-}
-
-/// Parquet row group writer API.
-/// Provides methods to access column writers in an iterator-like fashion, order is
-/// guaranteed to match the order of schema leaves (column descriptors).
-///
-/// All columns should be written sequentially; the main workflow is:
-/// - Request the next column using `next_column` method - this will return `None` if no
-/// more columns are available to write.
-/// - Once done writing a column, close column writer with `close_column` method - this
-/// will finalise column chunk metadata and update row group metrics.
-/// - Once all columns have been written, close row group writer with `close` method -
-/// it will return row group metadata and is no-op on already closed row group.
-pub trait RowGroupWriter {
- /// Returns the next column writer, if available; otherwise returns `None`.
- /// In case of any IO error or Thrift error, or if row group writer has already been
- /// closed returns `Err`.
- ///
- /// To request the next column writer, the previous one must be finalised and closed
- /// using `close_column`.
- fn next_column(&mut self) -> Result<Option<ColumnWriter>>;
-
- /// Closes column writer that was created using `next_column` method.
- /// This should be called before requesting the next column writer.
- fn close_column(&mut self, column_writer: ColumnWriter) -> Result<()>;
-
- /// Closes this row group writer and returns row group metadata.
- /// After calling this method row group writer must not be used.
- ///
- /// It is recommended to call this method before requesting another row group, but it
- /// will be closed automatically before returning a new row group.
- ///
- /// Can be called multiple times. In subsequent calls will result in no-op and return
- /// already created row group metadata.
- fn close(&mut self) -> Result<RowGroupMetaDataPtr>;
-}
-
-// ----------------------------------------------------------------------
-// Serialized impl for file & row group writers
-
-pub trait ParquetWriter: Write + Seek + TryClone {}
-impl<T: Write + Seek + TryClone> ParquetWriter for T {}
-
-/// A serialized implementation for Parquet [`FileWriter`].
-/// See documentation on file writer for more information.
-pub struct SerializedFileWriter<W: ParquetWriter> {
- buf: W,
- schema: TypePtr,
- descr: SchemaDescPtr,
- props: WriterPropertiesPtr,
- total_num_rows: i64,
- row_groups: Vec<RowGroupMetaDataPtr>,
- previous_writer_closed: bool,
- is_closed: bool,
-}
-
-impl<W: ParquetWriter> SerializedFileWriter<W> {
- /// Creates new file writer.
- pub fn new(
- mut buf: W,
- schema: TypePtr,
- properties: WriterPropertiesPtr,
- ) -> Result<Self> {
- Self::start_file(&mut buf)?;
- Ok(Self {
- buf,
- schema: schema.clone(),
- descr: Arc::new(SchemaDescriptor::new(schema)),
- props: properties,
- total_num_rows: 0,
- row_groups: Vec::new(),
- previous_writer_closed: true,
- is_closed: false,
- })
- }
-
- /// Writes magic bytes at the beginning of the file.
- fn start_file(buf: &mut W) -> Result<()> {
- buf.write_all(&PARQUET_MAGIC)?;
- Ok(())
- }
-
- /// Finalises active row group writer, otherwise no-op.
- fn finalise_row_group_writer(
- &mut self,
- mut row_group_writer: Box<dyn RowGroupWriter>,
- ) -> Result<()> {
- let row_group_metadata = row_group_writer.close()?;
- self.total_num_rows += row_group_metadata.num_rows();
- self.row_groups.push(row_group_metadata);
- Ok(())
- }
-
- /// Assembles and writes metadata at the end of the file.
- fn write_metadata(&mut self) -> Result<parquet::FileMetaData> {
- let file_metadata = parquet::FileMetaData {
- version: self.props.writer_version().as_num(),
- schema: types::to_thrift(self.schema.as_ref())?,
- num_rows: self.total_num_rows as i64,
- row_groups: self
- .row_groups
- .as_slice()
- .iter()
- .map(|v| v.to_thrift())
- .collect(),
- key_value_metadata: self.props.key_value_metadata().to_owned(),
- created_by: Some(self.props.created_by().to_owned()),
- column_orders: None,
- };
-
- // Write file metadata
- let start_pos = self.buf.seek(SeekFrom::Current(0))?;
- {
- let mut protocol = TCompactOutputProtocol::new(&mut self.buf);
- file_metadata.write_to_out_protocol(&mut protocol)?;
- protocol.flush()?;
- }
- let end_pos = self.buf.seek(SeekFrom::Current(0))?;
-
- // Write footer
- let mut footer_buffer: [u8; FOOTER_SIZE] = [0; FOOTER_SIZE];
- let metadata_len = (end_pos - start_pos) as i32;
- LittleEndian::write_i32(&mut footer_buffer, metadata_len);
- (&mut footer_buffer[4..]).write_all(&PARQUET_MAGIC)?;
- self.buf.write_all(&footer_buffer)?;
- Ok(file_metadata)
- }
-
- #[inline]
- fn assert_closed(&self) -> Result<()> {
- if self.is_closed {
- Err(general_err!("File writer is closed"))
- } else {
- Ok(())
- }
- }
-
- #[inline]
- fn assert_previous_writer_closed(&self) -> Result<()> {
- if !self.previous_writer_closed {
- Err(general_err!("Previous row group writer was not closed"))
- } else {
- Ok(())
- }
- }
-}
-
-impl<W: 'static + ParquetWriter> FileWriter for SerializedFileWriter<W> {
- #[inline]
- fn next_row_group(&mut self) -> Result<Box<dyn RowGroupWriter>> {
- self.assert_closed()?;
- self.assert_previous_writer_closed()?;
- let row_group_writer = SerializedRowGroupWriter::new(
- self.descr.clone(),
- self.props.clone(),
- &self.buf,
- );
- self.previous_writer_closed = false;
- Ok(Box::new(row_group_writer))
- }
-
- #[inline]
- fn close_row_group(
- &mut self,
- row_group_writer: Box<dyn RowGroupWriter>,
- ) -> Result<()> {
- self.assert_closed()?;
- let res = self.finalise_row_group_writer(row_group_writer);
- self.previous_writer_closed = res.is_ok();
- res
- }
-
- #[inline]
- fn close(&mut self) -> Result<parquet::FileMetaData> {
- self.assert_closed()?;
- self.assert_previous_writer_closed()?;
- let metadata = self.write_metadata()?;
- self.is_closed = true;
- Ok(metadata)
- }
-}
-
-/// A serialized implementation for Parquet [`RowGroupWriter`].
-/// Coordinates writing of a row group with column writers.
-/// See documentation on row group writer for more information.
-pub struct SerializedRowGroupWriter<W: ParquetWriter> {
- descr: SchemaDescPtr,
- props: WriterPropertiesPtr,
- buf: W,
- total_rows_written: Option<u64>,
- total_bytes_written: u64,
- column_index: usize,
- previous_writer_closed: bool,
- row_group_metadata: Option<RowGroupMetaDataPtr>,
- column_chunks: Vec<ColumnChunkMetaData>,
-}
-
-impl<W: 'static + ParquetWriter> SerializedRowGroupWriter<W> {
- pub fn new(
- schema_descr: SchemaDescPtr,
- properties: WriterPropertiesPtr,
- buf: &W,
- ) -> Self {
- let num_columns = schema_descr.num_columns();
- Self {
- descr: schema_descr,
- props: properties,
- buf: buf.try_clone().unwrap(),
- total_rows_written: None,
- total_bytes_written: 0,
- column_index: 0,
- previous_writer_closed: true,
- row_group_metadata: None,
- column_chunks: Vec::with_capacity(num_columns),
- }
- }
-
- /// Checks and finalises current column writer.
- fn finalise_column_writer(&mut self, writer: ColumnWriter) -> Result<()> {
- let (bytes_written, rows_written, metadata) = match writer {
- ColumnWriter::BoolColumnWriter(typed) => typed.close()?,
- ColumnWriter::Int32ColumnWriter(typed) => typed.close()?,
- ColumnWriter::Int64ColumnWriter(typed) => typed.close()?,
- ColumnWriter::Int96ColumnWriter(typed) => typed.close()?,
- ColumnWriter::FloatColumnWriter(typed) => typed.close()?,
- ColumnWriter::DoubleColumnWriter(typed) => typed.close()?,
- ColumnWriter::ByteArrayColumnWriter(typed) => typed.close()?,
- ColumnWriter::FixedLenByteArrayColumnWriter(typed) => typed.close()?,
- };
-
- // Update row group writer metrics
- self.total_bytes_written += bytes_written;
- self.column_chunks.push(metadata);
- if let Some(rows) = self.total_rows_written {
- if rows != rows_written {
- return Err(general_err!(
- "Incorrect number of rows, expected {} != {} rows",
- rows,
- rows_written
- ));
- }
- } else {
- self.total_rows_written = Some(rows_written);
- }
-
- Ok(())
- }
-
- #[inline]
- fn assert_closed(&self) -> Result<()> {
- if self.row_group_metadata.is_some() {
- Err(general_err!("Row group writer is closed"))
- } else {
- Ok(())
- }
- }
-
- #[inline]
- fn assert_previous_writer_closed(&self) -> Result<()> {
- if !self.previous_writer_closed {
- Err(general_err!("Previous column writer was not closed"))
- } else {
- Ok(())
- }
- }
-}
-
-impl<W: 'static + ParquetWriter> RowGroupWriter for SerializedRowGroupWriter<W> {
- #[inline]
- fn next_column(&mut self) -> Result<Option<ColumnWriter>> {
- self.assert_closed()?;
- self.assert_previous_writer_closed()?;
-
- if self.column_index >= self.descr.num_columns() {
- return Ok(None);
- }
- let sink = FileSink::new(&self.buf);
- let page_writer = Box::new(SerializedPageWriter::new(sink));
- let column_writer = get_column_writer(
- self.descr.column(self.column_index),
- self.props.clone(),
- page_writer,
- );
- self.column_index += 1;
- self.previous_writer_closed = false;
-
- Ok(Some(column_writer))
- }
-
- #[inline]
- fn close_column(&mut self, column_writer: ColumnWriter) -> Result<()> {
- let res = self.finalise_column_writer(column_writer);
- self.previous_writer_closed = res.is_ok();
- res
- }
-
- #[inline]
- fn close(&mut self) -> Result<RowGroupMetaDataPtr> {
- if self.row_group_metadata.is_none() {
- self.assert_previous_writer_closed()?;
-
- let column_chunks = std::mem::take(&mut self.column_chunks);
- let row_group_metadata = RowGroupMetaData::builder(self.descr.clone())
- .set_column_metadata(column_chunks)
- .set_total_byte_size(self.total_bytes_written as i64)
- .set_num_rows(self.total_rows_written.unwrap_or(0) as i64)
- .build()?;
-
- self.row_group_metadata = Some(Arc::new(row_group_metadata));
- }
-
- let metadata = self.row_group_metadata.as_ref().unwrap().clone();
- Ok(metadata)
- }
-}
-
-/// A serialized implementation for Parquet [`PageWriter`].
-/// Writes and serializes pages and metadata into output stream.
-///
-/// `SerializedPageWriter` should not be used after calling `close()`.
-pub struct SerializedPageWriter<T: Write + Position> {
- sink: T,
-}
-
-impl<T: Write + Position> SerializedPageWriter<T> {
- /// Creates new page writer.
- pub fn new(sink: T) -> Self {
- Self { sink }
- }
-
- /// Serializes page header into Thrift.
- /// Returns number of bytes that have been written into the sink.
- #[inline]
- fn serialize_page_header(&mut self, header: parquet::PageHeader) -> Result<usize> {
- let start_pos = self.sink.pos();
- {
- let mut protocol = TCompactOutputProtocol::new(&mut self.sink);
- header.write_to_out_protocol(&mut protocol)?;
- protocol.flush()?;
- }
- Ok((self.sink.pos() - start_pos) as usize)
- }
-
- /// Serializes column chunk into Thrift.
- /// Returns Ok() if there are not errors serializing and writing data into the sink.
- #[inline]
- fn serialize_column_chunk(&mut self, chunk: parquet::ColumnChunk) -> Result<()> {
- let mut protocol = TCompactOutputProtocol::new(&mut self.sink);
- chunk.write_to_out_protocol(&mut protocol)?;
- protocol.flush()?;
- Ok(())
- }
-}
-
-impl<T: Write + Position> PageWriter for SerializedPageWriter<T> {
- fn write_page(&mut self, page: CompressedPage) -> Result<PageWriteSpec> {
- let uncompressed_size = page.uncompressed_size();
- let compressed_size = page.compressed_size();
- let num_values = page.num_values();
- let encoding = page.encoding();
- let page_type = page.page_type();
-
- let mut page_header = parquet::PageHeader {
- type_: page_type.into(),
- uncompressed_page_size: uncompressed_size as i32,
- compressed_page_size: compressed_size as i32,
- // TODO: Add support for crc checksum
- crc: None,
- data_page_header: None,
- index_page_header: None,
- dictionary_page_header: None,
- data_page_header_v2: None,
- };
-
- match *page.compressed_page() {
- Page::DataPage {
- def_level_encoding,
- rep_level_encoding,
- ref statistics,
- ..
- } => {
- let data_page_header = parquet::DataPageHeader {
- num_values: num_values as i32,
- encoding: encoding.into(),
- definition_level_encoding: def_level_encoding.into(),
- repetition_level_encoding: rep_level_encoding.into(),
- statistics: statistics_to_thrift(statistics.as_ref()),
- };
- page_header.data_page_header = Some(data_page_header);
- }
- Page::DataPageV2 {
- num_nulls,
- num_rows,
- def_levels_byte_len,
- rep_levels_byte_len,
- is_compressed,
- ref statistics,
- ..
- } => {
- let data_page_header_v2 = parquet::DataPageHeaderV2 {
- num_values: num_values as i32,
- num_nulls: num_nulls as i32,
- num_rows: num_rows as i32,
- encoding: encoding.into(),
- definition_levels_byte_length: def_levels_byte_len as i32,
- repetition_levels_byte_length: rep_levels_byte_len as i32,
- is_compressed: Some(is_compressed),
- statistics: statistics_to_thrift(statistics.as_ref()),
- };
- page_header.data_page_header_v2 = Some(data_page_header_v2);
- }
- Page::DictionaryPage { is_sorted, .. } => {
- let dictionary_page_header = parquet::DictionaryPageHeader {
- num_values: num_values as i32,
- encoding: encoding.into(),
- is_sorted: Some(is_sorted),
- };
- page_header.dictionary_page_header = Some(dictionary_page_header);
- }
- }
-
- let start_pos = self.sink.pos();
-
- let header_size = self.serialize_page_header(page_header)?;
- self.sink.write_all(page.data())?;
-
- let mut spec = PageWriteSpec::new();
- spec.page_type = page_type;
- spec.uncompressed_size = uncompressed_size + header_size;
- spec.compressed_size = compressed_size + header_size;
- spec.offset = start_pos;
- spec.bytes_written = self.sink.pos() - start_pos;
- // Number of values is incremented for data pages only
- if page_type == PageType::DATA_PAGE || page_type == PageType::DATA_PAGE_V2 {
- spec.num_values = num_values;
- }
-
- Ok(spec)
- }
-
- fn write_metadata(&mut self, metadata: &ColumnChunkMetaData) -> Result<()> {
- self.serialize_column_chunk(metadata.to_thrift())
- }
-
- fn close(&mut self) -> Result<()> {
- self.sink.flush()?;
- Ok(())
- }
-}
-
-#[cfg(test)]
-mod tests {
- use super::*;
-
- use std::{fs::File, io::Cursor};
-
- use crate::basic::{Compression, Encoding, IntType, LogicalType, Repetition, Type};
- use crate::column::page::PageReader;
- use crate::compression::{create_codec, Codec};
- use crate::file::{
- properties::{WriterProperties, WriterVersion},
- reader::{FileReader, SerializedFileReader, SerializedPageReader},
- statistics::{from_thrift, to_thrift, Statistics},
- };
- use crate::record::RowAccessor;
- use crate::util::{memory::ByteBufferPtr, test_common::get_temp_file};
-
- #[test]
- fn test_file_writer_error_after_close() {
- let file = get_temp_file("test_file_writer_error_after_close", &[]);
- let schema = Arc::new(types::Type::group_type_builder("schema").build().unwrap());
- let props = Arc::new(WriterProperties::builder().build());
- let mut writer = SerializedFileWriter::new(file, schema, props).unwrap();
- writer.close().unwrap();
- {
- let res = writer.next_row_group();
- assert!(res.is_err());
- if let Err(err) = res {
- assert_eq!(format!("{}", err), "Parquet error: File writer is closed");
- }
- }
- {
- let res = writer.close();
- assert!(res.is_err());
- if let Err(err) = res {
- assert_eq!(format!("{}", err), "Parquet error: File writer is closed");
- }
- }
- }
-
- #[test]
- fn test_row_group_writer_error_after_close() {
- let file = get_temp_file("test_file_writer_row_group_error_after_close", &[]);
- let schema = Arc::new(types::Type::group_type_builder("schema").build().unwrap());
- let props = Arc::new(WriterProperties::builder().build());
- let mut writer = SerializedFileWriter::new(file, schema, props).unwrap();
- let mut row_group_writer = writer.next_row_group().unwrap();
- row_group_writer.close().unwrap();
-
- let res = row_group_writer.next_column();
- assert!(res.is_err());
- if let Err(err) = res {
- assert_eq!(
- format!("{}", err),
- "Parquet error: Row group writer is closed"
- );
- }
- }
-
- #[test]
- fn test_row_group_writer_error_not_all_columns_written() {
- let file =
- get_temp_file("test_row_group_writer_error_not_all_columns_written", &[]);
- let schema = Arc::new(
- types::Type::group_type_builder("schema")
- .with_fields(&mut vec![Arc::new(
- types::Type::primitive_type_builder("col1", Type::INT32)
- .build()
- .unwrap(),
- )])
- .build()
- .unwrap(),
- );
- let props = Arc::new(WriterProperties::builder().build());
- let mut writer = SerializedFileWriter::new(file, schema, props).unwrap();
- let mut row_group_writer = writer.next_row_group().unwrap();
- let res = row_group_writer.close();
- assert!(res.is_err());
- if let Err(err) = res {
- assert_eq!(
- format!("{}", err),
- "Parquet error: Column length mismatch: 1 != 0"
- );
- }
- }
-
- #[test]
- fn test_row_group_writer_num_records_mismatch() {
- let file = get_temp_file("test_row_group_writer_num_records_mismatch", &[]);
- let schema = Arc::new(
- types::Type::group_type_builder("schema")
- .with_fields(&mut vec![
- Arc::new(
- types::Type::primitive_type_builder("col1", Type::INT32)
- .with_repetition(Repetition::REQUIRED)
- .build()
- .unwrap(),
- ),
- Arc::new(
- types::Type::primitive_type_builder("col2", Type::INT32)
- .with_repetition(Repetition::REQUIRED)
- .build()
- .unwrap(),
- ),
- ])
- .build()
- .unwrap(),
- );
- let props = Arc::new(WriterProperties::builder().build());
- let mut writer = SerializedFileWriter::new(file, schema, props).unwrap();
- let mut row_group_writer = writer.next_row_group().unwrap();
-
- let mut col_writer = row_group_writer.next_column().unwrap().unwrap();
- if let ColumnWriter::Int32ColumnWriter(ref mut typed) = col_writer {
- typed.write_batch(&[1, 2, 3], None, None).unwrap();
- }
- row_group_writer.close_column(col_writer).unwrap();
-
- let mut col_writer = row_group_writer.next_column().unwrap().unwrap();
- if let ColumnWriter::Int32ColumnWriter(ref mut typed) = col_writer {
- typed.write_batch(&[1, 2], None, None).unwrap();
- }
-
- let res = row_group_writer.close_column(col_writer);
- assert!(res.is_err());
- if let Err(err) = res {
- assert_eq!(
- format!("{}", err),
- "Parquet error: Incorrect number of rows, expected 3 != 2 rows"
- );
- }
- }
-
- #[test]
- fn test_file_writer_empty_file() {
- let file = get_temp_file("test_file_writer_write_empty_file", &[]);
-
- let schema = Arc::new(
- types::Type::group_type_builder("schema")
- .with_fields(&mut vec![Arc::new(
- types::Type::primitive_type_builder("col1", Type::INT32)
- .build()
- .unwrap(),
- )])
- .build()
- .unwrap(),
- );
- let props = Arc::new(WriterProperties::builder().build());
- let mut writer =
- SerializedFileWriter::new(file.try_clone().unwrap(), schema, props).unwrap();
- writer.close().unwrap();
-
- let reader = SerializedFileReader::new(file).unwrap();
- assert_eq!(reader.get_row_iter(None).unwrap().count(), 0);
- }
-
- #[test]
- fn test_file_writer_with_metadata() {
- let file = get_temp_file("test_file_writer_write_with_metadata", &[]);
-
- let schema = Arc::new(
- types::Type::group_type_builder("schema")
- .with_fields(&mut vec![Arc::new(
- types::Type::primitive_type_builder("col1", Type::INT32)
- .build()
- .unwrap(),
- )])
- .build()
- .unwrap(),
- );
- let props = Arc::new(
- WriterProperties::builder()
- .set_key_value_metadata(Some(vec![KeyValue::new(
- "key".to_string(),
- "value".to_string(),
- )]))
- .build(),
- );
- let mut writer =
- SerializedFileWriter::new(file.try_clone().unwrap(), schema, props).unwrap();
- writer.close().unwrap();
-
- let reader = SerializedFileReader::new(file).unwrap();
- assert_eq!(
- reader
- .metadata()
- .file_metadata()
- .key_value_metadata()
- .to_owned()
- .unwrap()
- .len(),
- 1
- );
- }
-
- #[test]
- fn test_file_writer_v2_with_metadata() {
- let file = get_temp_file("test_file_writer_v2_write_with_metadata", &[]);
- let field_logical_type = Some(LogicalType::INTEGER(IntType {
- bit_width: 8,
- is_signed: false,
- }));
- let field = Arc::new(
- types::Type::primitive_type_builder("col1", Type::INT32)
- .with_logical_type(field_logical_type.clone())
- .with_converted_type(field_logical_type.into())
- .build()
- .unwrap(),
- );
- let schema = Arc::new(
- types::Type::group_type_builder("schema")
- .with_fields(&mut vec![field.clone()])
- .build()
- .unwrap(),
- );
- let props = Arc::new(
- WriterProperties::builder()
- .set_key_value_metadata(Some(vec![KeyValue::new(
- "key".to_string(),
- "value".to_string(),
- )]))
- .set_writer_version(WriterVersion::PARQUET_2_0)
- .build(),
- );
- let mut writer =
- SerializedFileWriter::new(file.try_clone().unwrap(), schema, props).unwrap();
- writer.close().unwrap();
-
- let reader = SerializedFileReader::new(file).unwrap();
-
- assert_eq!(
- reader
- .metadata()
- .file_metadata()
- .key_value_metadata()
- .to_owned()
- .unwrap()
- .len(),
- 1
- );
-
- // ARROW-11803: Test that the converted and logical types have been populated
- let fields = reader.metadata().file_metadata().schema().get_fields();
- assert_eq!(fields.len(), 1);
- let read_field = fields.get(0).unwrap();
- assert_eq!(read_field, &field);
- }
-
- #[test]
- fn test_file_writer_empty_row_groups() {
- let file = get_temp_file("test_file_writer_write_empty_row_groups", &[]);
- test_file_roundtrip(file, vec![]);
- }
-
- #[test]
- fn test_file_writer_single_row_group() {
- let file = get_temp_file("test_file_writer_write_single_row_group", &[]);
- test_file_roundtrip(file, vec![vec![1, 2, 3, 4, 5]]);
- }
-
- #[test]
- fn test_file_writer_multiple_row_groups() {
- let file = get_temp_file("test_file_writer_write_multiple_row_groups", &[]);
- test_file_roundtrip(
- file,
- vec![
- vec![1, 2, 3, 4, 5],
- vec![1, 2, 3],
- vec![1],
- vec![1, 2, 3, 4, 5, 6],
- ],
- );
- }
-
- #[test]
- fn test_file_writer_multiple_large_row_groups() {
- let file = get_temp_file("test_file_writer_multiple_large_row_groups", &[]);
- test_file_roundtrip(
- file,
- vec![vec![123; 1024], vec![124; 1000], vec![125; 15], vec![]],
- );
- }
-
- #[test]
- fn test_page_writer_data_pages() {
- let pages = vec![
- Page::DataPage {
- buf: ByteBufferPtr::new(vec![1, 2, 3, 4, 5, 6, 7, 8]),
- num_values: 10,
- encoding: Encoding::DELTA_BINARY_PACKED,
- def_level_encoding: Encoding::RLE,
- rep_level_encoding: Encoding::RLE,
- statistics: Some(Statistics::int32(Some(1), Some(3), None, 7, true)),
- },
- Page::DataPageV2 {
- buf: ByteBufferPtr::new(vec![4; 128]),
- num_values: 10,
- encoding: Encoding::DELTA_BINARY_PACKED,
- num_nulls: 2,
- num_rows: 12,
- def_levels_byte_len: 24,
- rep_levels_byte_len: 32,
- is_compressed: false,
- statistics: Some(Statistics::int32(Some(1), Some(3), None, 7, true)),
- },
- ];
-
- test_page_roundtrip(&pages[..], Compression::SNAPPY, Type::INT32);
- test_page_roundtrip(&pages[..], Compression::UNCOMPRESSED, Type::INT32);
- }
-
- #[test]
- fn test_page_writer_dict_pages() {
- let pages = vec![
- Page::DictionaryPage {
- buf: ByteBufferPtr::new(vec![1, 2, 3, 4, 5]),
- num_values: 5,
- encoding: Encoding::RLE_DICTIONARY,
- is_sorted: false,
- },
- Page::DataPage {
- buf: ByteBufferPtr::new(vec![1, 2, 3, 4, 5, 6, 7, 8]),
- num_values: 10,
- encoding: Encoding::DELTA_BINARY_PACKED,
- def_level_encoding: Encoding::RLE,
- rep_level_encoding: Encoding::RLE,
- statistics: Some(Statistics::int32(Some(1), Some(3), None, 7, true)),
- },
- Page::DataPageV2 {
- buf: ByteBufferPtr::new(vec![4; 128]),
- num_values: 10,
- encoding: Encoding::DELTA_BINARY_PACKED,
- num_nulls: 2,
- num_rows: 12,
- def_levels_byte_len: 24,
- rep_levels_byte_len: 32,
- is_compressed: false,
- statistics: None,
- },
- ];
-
- test_page_roundtrip(&pages[..], Compression::SNAPPY, Type::INT32);
- test_page_roundtrip(&pages[..], Compression::UNCOMPRESSED, Type::INT32);
- }
-
- /// Tests writing and reading pages.
- /// Physical type is for statistics only, should match any defined statistics type in
- /// pages.
- fn test_page_roundtrip(pages: &[Page], codec: Compression, physical_type: Type) {
- let mut compressed_pages = vec![];
- let mut total_num_values = 0i64;
- let mut compressor = create_codec(codec).unwrap();
-
- for page in pages {
- let uncompressed_len = page.buffer().len();
-
- let compressed_page = match *page {
- Page::DataPage {
- ref buf,
- num_values,
- encoding,
- def_level_encoding,
- rep_level_encoding,
- ref statistics,
- } => {
- total_num_values += num_values as i64;
- let output_buf = compress_helper(compressor.as_mut(), buf.data());
-
- Page::DataPage {
- buf: ByteBufferPtr::new(output_buf),
- num_values,
- encoding,
- def_level_encoding,
- rep_level_encoding,
- statistics: from_thrift(
- physical_type,
- to_thrift(statistics.as_ref()),
- ),
- }
- }
- Page::DataPageV2 {
- ref buf,
- num_values,
- encoding,
- num_nulls,
- num_rows,
- def_levels_byte_len,
- rep_levels_byte_len,
- ref statistics,
- ..
- } => {
- total_num_values += num_values as i64;
- let offset = (def_levels_byte_len + rep_levels_byte_len) as usize;
- let cmp_buf =
- compress_helper(compressor.as_mut(), &buf.data()[offset..]);
- let mut output_buf = Vec::from(&buf.data()[..offset]);
- output_buf.extend_from_slice(&cmp_buf[..]);
-
- Page::DataPageV2 {
- buf: ByteBufferPtr::new(output_buf),
- num_values,
- encoding,
- num_nulls,
- num_rows,
- def_levels_byte_len,
- rep_levels_byte_len,
- is_compressed: compressor.is_some(),
- statistics: from_thrift(
- physical_type,
- to_thrift(statistics.as_ref()),
- ),
- }
- }
- Page::DictionaryPage {
- ref buf,
- num_values,
- encoding,
- is_sorted,
- } => {
- let output_buf = compress_helper(compressor.as_mut(), buf.data());
-
- Page::DictionaryPage {
- buf: ByteBufferPtr::new(output_buf),
- num_values,
- encoding,
- is_sorted,
- }
- }
- };
-
- let compressed_page = CompressedPage::new(compressed_page, uncompressed_len);
- compressed_pages.push(compressed_page);
- }
-
- let mut buffer: Vec<u8> = vec![];
- let mut result_pages: Vec<Page> = vec![];
- {
- let cursor = Cursor::new(&mut buffer);
- let mut page_writer = SerializedPageWriter::new(cursor);
-
- for page in compressed_pages {
- page_writer.write_page(page).unwrap();
- }
- page_writer.close().unwrap();
- }
- {
- let mut page_reader = SerializedPageReader::new(
- Cursor::new(&buffer),
- total_num_values,
- codec,
- physical_type,
- )
- .unwrap();
-
- while let Some(page) = page_reader.get_next_page().unwrap() {
- result_pages.push(page);
- }
- }
-
- assert_eq!(result_pages.len(), pages.len());
- for i in 0..result_pages.len() {
- assert_page(&result_pages[i], &pages[i]);
- }
- }
-
- /// Helper function to compress a slice
- fn compress_helper(compressor: Option<&mut Box<dyn Codec>>, data: &[u8]) -> Vec<u8> {
- let mut output_buf = vec![];
- if let Some(cmpr) = compressor {
- cmpr.compress(data, &mut output_buf).unwrap();
- } else {
- output_buf.extend_from_slice(data);
- }
- output_buf
- }
-
- /// Check if pages match.
- fn assert_page(left: &Page, right: &Page) {
- assert_eq!(left.page_type(), right.page_type());
- assert_eq!(left.buffer().data(), right.buffer().data());
- assert_eq!(left.num_values(), right.num_values());
- assert_eq!(left.encoding(), right.encoding());
- assert_eq!(to_thrift(left.statistics()), to_thrift(right.statistics()));
- }
-
- /// File write-read roundtrip.
- /// `data` consists of arrays of values for each row group.
- fn test_file_roundtrip(file: File, data: Vec<Vec<i32>>) {
- let schema = Arc::new(
- types::Type::group_type_builder("schema")
- .with_fields(&mut vec![Arc::new(
- types::Type::primitive_type_builder("col1", Type::INT32)
- .with_repetition(Repetition::REQUIRED)
- .build()
- .unwrap(),
- )])
- .build()
- .unwrap(),
- );
- let props = Arc::new(WriterProperties::builder().build());
- let mut file_writer = assert_send(
- SerializedFileWriter::new(file.try_clone().unwrap(), schema, props).unwrap(),
- );
- let mut rows: i64 = 0;
-
- for subset in &data {
- let mut row_group_writer = file_writer.next_row_group().unwrap();
- let col_writer = row_group_writer.next_column().unwrap();
- if let Some(mut writer) = col_writer {
- match writer {
- ColumnWriter::Int32ColumnWriter(ref mut typed) => {
- rows +=
- typed.write_batch(&subset[..], None, None).unwrap() as i64;
- }
- _ => {
- unimplemented!();
- }
- }
- row_group_writer.close_column(writer).unwrap();
- }
- file_writer.close_row_group(row_group_writer).unwrap();
- }
-
- file_writer.close().unwrap();
-
- let reader = assert_send(SerializedFileReader::new(file).unwrap());
- assert_eq!(reader.num_row_groups(), data.len());
- assert_eq!(
- reader.metadata().file_metadata().num_rows(),
- rows,
- "row count in metadata not equal to number of rows written"
- );
- for i in 0..reader.num_row_groups() {
- let row_group_reader = reader.get_row_group(i).unwrap();
- let iter = row_group_reader.get_row_iter(None).unwrap();
- let res = iter
- .map(|elem| elem.get_int(0).unwrap())
- .collect::<Vec<i32>>();
- assert_eq!(res, data[i]);
- }
- }
-
- fn assert_send<T: Send>(t: T) -> T {
- t
- }
-
- #[test]
- fn test_bytes_writer_empty_row_groups() {
- test_bytes_roundtrip(vec![]);
- }
-
- #[test]
- fn test_bytes_writer_single_row_group() {
- test_bytes_roundtrip(vec![vec![1, 2, 3, 4, 5]]);
- }
-
- #[test]
- fn test_bytes_writer_multiple_row_groups() {
- test_bytes_roundtrip(vec![
- vec![1, 2, 3, 4, 5],
- vec![1, 2, 3],
- vec![1],
- vec![1, 2, 3, 4, 5, 6],
- ]);
- }
-
- fn test_bytes_roundtrip(data: Vec<Vec<i32>>) {
- let cursor = InMemoryWriteableCursor::default();
-
- let schema = Arc::new(
- types::Type::group_type_builder("schema")
- .with_fields(&mut vec![Arc::new(
- types::Type::primitive_type_builder("col1", Type::INT32)
- .with_repetition(Repetition::REQUIRED)
- .build()
- .unwrap(),
- )])
- .build()
- .unwrap(),
- );
-
- let mut rows: i64 = 0;
- {
- let props = Arc::new(WriterProperties::builder().build());
- let mut writer =
- SerializedFileWriter::new(cursor.clone(), schema, props).unwrap();
-
- for subset in &data {
- let mut row_group_writer = writer.next_row_group().unwrap();
- let col_writer = row_group_writer.next_column().unwrap();
- if let Some(mut writer) = col_writer {
- match writer {
- ColumnWriter::Int32ColumnWriter(ref mut typed) => {
- rows += typed.write_batch(&subset[..], None, None).unwrap()
- as i64;
- }
- _ => {
- unimplemented!();
- }
- }
- row_group_writer.close_column(writer).unwrap();
- }
- writer.close_row_group(row_group_writer).unwrap();
- }
-
- writer.close().unwrap();
- }
-
- let buffer = cursor.into_inner().unwrap();
-
- let reading_cursor = crate::file::serialized_reader::SliceableCursor::new(buffer);
- let reader = SerializedFileReader::new(reading_cursor).unwrap();
-
- assert_eq!(reader.num_row_groups(), data.len());
- assert_eq!(
- reader.metadata().file_metadata().num_rows(),
- rows,
- "row count in metadata not equal to number of rows written"
- );
- for i in 0..reader.num_row_groups() {
- let row_group_reader = reader.get_row_group(i).unwrap();
- let iter = row_group_reader.get_row_iter(None).unwrap();
- let res = iter
- .map(|elem| elem.get_int(0).unwrap())
- .collect::<Vec<i32>>();
- assert_eq!(res, data[i]);
- }
- }
-}
diff --git a/parquet/src/lib.rs b/parquet/src/lib.rs
deleted file mode 100644
index 900e2b5..0000000
--- a/parquet/src/lib.rs
+++ /dev/null
@@ -1,57 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#![allow(incomplete_features)]
-#![allow(dead_code)]
-#![allow(non_camel_case_types)]
-#![allow(
- clippy::approx_constant,
- clippy::cast_ptr_alignment,
- clippy::float_cmp,
- clippy::float_equality_without_abs,
- clippy::from_over_into,
- clippy::many_single_char_names,
- clippy::needless_range_loop,
- clippy::new_without_default,
- clippy::or_fun_call,
- clippy::same_item_push,
- clippy::too_many_arguments,
- clippy::transmute_ptr_to_ptr,
- clippy::upper_case_acronyms,
- clippy::vec_init_then_push
-)]
-
-#[macro_use]
-pub mod errors;
-pub mod basic;
-#[macro_use]
-pub mod data_type;
-
-// Exported for external use, such as benchmarks
-pub use self::encodings::{decoding, encoding};
-pub use self::util::memory;
-
-#[macro_use]
-pub mod util;
-#[cfg(any(feature = "arrow", test))]
-pub mod arrow;
-pub mod column;
-pub mod compression;
-mod encodings;
-pub mod file;
-pub mod record;
-pub mod schema;
diff --git a/parquet/src/record/api.rs b/parquet/src/record/api.rs
deleted file mode 100644
index 411016e..0000000
--- a/parquet/src/record/api.rs
+++ /dev/null
@@ -1,1846 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-//! Contains Row enum that is used to represent record in Rust.
-
-use std::fmt;
-
-use chrono::{TimeZone, Utc};
-use num_bigint::{BigInt, Sign};
-
-use crate::basic::{ConvertedType, Type as PhysicalType};
-use crate::data_type::{ByteArray, Decimal, Int96};
-use crate::errors::{ParquetError, Result};
-use crate::schema::types::ColumnDescPtr;
-
-#[cfg(feature = "cli")]
-use serde_json::Value;
-
-/// Macro as a shortcut to generate 'not yet implemented' panic error.
-macro_rules! nyi {
- ($column_descr:ident, $value:ident) => {{
- unimplemented!(
- "Conversion for physical type {}, converted type {}, value {:?}",
- $column_descr.physical_type(),
- $column_descr.converted_type(),
- $value
- );
- }};
-}
-
-/// `Row` represents a nested Parquet record.
-#[derive(Clone, Debug, PartialEq)]
-pub struct Row {
- fields: Vec<(String, Field)>,
-}
-
-#[allow(clippy::len_without_is_empty)]
-impl Row {
- /// Get the number of fields in this row.
- pub fn len(&self) -> usize {
- self.fields.len()
- }
-
- /// Get an iterator to go through all columns in the row.
- ///
- /// # Example
- ///
- /// ```no_run
- /// use std::fs::File;
- /// use parquet::record::Row;
- /// use parquet::file::reader::{FileReader, SerializedFileReader};
- ///
- /// let file = File::open("/path/to/file").unwrap();
- /// let reader = SerializedFileReader::new(file).unwrap();
- /// let row: Row = reader.get_row_iter(None).unwrap().next().unwrap();
- /// for (idx, (name, field)) in row.get_column_iter().enumerate() {
- /// println!("column index: {}, column name: {}, column value: {}", idx, name, field);
- /// }
- /// ```
- pub fn get_column_iter(&self) -> RowColumnIter {
- RowColumnIter {
- fields: &self.fields,
- curr: 0,
- count: self.fields.len(),
- }
- }
-
- #[cfg(feature = "cli")]
- pub fn to_json_value(&self) -> Value {
- Value::Object(
- self.fields
- .iter()
- .map(|(key, field)| (key.to_owned(), field.to_json_value()))
- .collect(),
- )
- }
-}
-
-pub struct RowColumnIter<'a> {
- fields: &'a Vec<(String, Field)>,
- curr: usize,
- count: usize,
-}
-
-impl<'a> Iterator for RowColumnIter<'a> {
- type Item = (&'a String, &'a Field);
-
- fn next(&mut self) -> Option<Self::Item> {
- let idx = self.curr;
- if idx >= self.count {
- return None;
- }
- self.curr += 1;
- Some((&self.fields[idx].0, &self.fields[idx].1))
- }
-}
-
-/// Trait for type-safe convenient access to fields within a Row.
-pub trait RowAccessor {
- fn get_bool(&self, i: usize) -> Result<bool>;
- fn get_byte(&self, i: usize) -> Result<i8>;
- fn get_short(&self, i: usize) -> Result<i16>;
- fn get_int(&self, i: usize) -> Result<i32>;
- fn get_long(&self, i: usize) -> Result<i64>;
- fn get_ubyte(&self, i: usize) -> Result<u8>;
- fn get_ushort(&self, i: usize) -> Result<u16>;
- fn get_uint(&self, i: usize) -> Result<u32>;
- fn get_ulong(&self, i: usize) -> Result<u64>;
- fn get_float(&self, i: usize) -> Result<f32>;
- fn get_double(&self, i: usize) -> Result<f64>;
- fn get_timestamp_millis(&self, i: usize) -> Result<u64>;
- fn get_timestamp_micros(&self, i: usize) -> Result<u64>;
- fn get_decimal(&self, i: usize) -> Result<&Decimal>;
- fn get_string(&self, i: usize) -> Result<&String>;
- fn get_bytes(&self, i: usize) -> Result<&ByteArray>;
- fn get_group(&self, i: usize) -> Result<&Row>;
- fn get_list(&self, i: usize) -> Result<&List>;
- fn get_map(&self, i: usize) -> Result<&Map>;
-}
-
-/// Trait for formating fields within a Row.
-pub trait RowFormatter {
- fn fmt(&self, i: usize) -> &dyn fmt::Display;
-}
-
-/// Macro to generate type-safe get_xxx methods for primitive types,
-/// e.g. `get_bool`, `get_short`.
-macro_rules! row_primitive_accessor {
- ($METHOD:ident, $VARIANT:ident, $TY:ty) => {
- fn $METHOD(&self, i: usize) -> Result<$TY> {
- match self.fields[i].1 {
- Field::$VARIANT(v) => Ok(v),
- _ => Err(general_err!(
- "Cannot access {} as {}",
- self.fields[i].1.get_type_name(),
- stringify!($VARIANT)
- )),
- }
- }
- };
-}
-
-/// Macro to generate type-safe get_xxx methods for reference types,
-/// e.g. `get_list`, `get_map`.
-macro_rules! row_complex_accessor {
- ($METHOD:ident, $VARIANT:ident, $TY:ty) => {
- fn $METHOD(&self, i: usize) -> Result<&$TY> {
- match self.fields[i].1 {
- Field::$VARIANT(ref v) => Ok(v),
- _ => Err(general_err!(
- "Cannot access {} as {}",
- self.fields[i].1.get_type_name(),
- stringify!($VARIANT)
- )),
- }
- }
- };
-}
-
-impl RowFormatter for Row {
- /// Get Display reference for a given field.
- fn fmt(&self, i: usize) -> &dyn fmt::Display {
- &self.fields[i].1
- }
-}
-
-impl RowAccessor for Row {
- row_primitive_accessor!(get_bool, Bool, bool);
-
- row_primitive_accessor!(get_byte, Byte, i8);
-
- row_primitive_accessor!(get_short, Short, i16);
-
- row_primitive_accessor!(get_int, Int, i32);
-
- row_primitive_accessor!(get_long, Long, i64);
-
- row_primitive_accessor!(get_ubyte, UByte, u8);
-
- row_primitive_accessor!(get_ushort, UShort, u16);
-
- row_primitive_accessor!(get_uint, UInt, u32);
-
- row_primitive_accessor!(get_ulong, ULong, u64);
-
- row_primitive_accessor!(get_float, Float, f32);
-
- row_primitive_accessor!(get_double, Double, f64);
-
- row_primitive_accessor!(get_timestamp_millis, TimestampMillis, u64);
-
- row_primitive_accessor!(get_timestamp_micros, TimestampMicros, u64);
-
- row_complex_accessor!(get_decimal, Decimal, Decimal);
-
- row_complex_accessor!(get_string, Str, String);
-
- row_complex_accessor!(get_bytes, Bytes, ByteArray);
-
- row_complex_accessor!(get_group, Group, Row);
-
- row_complex_accessor!(get_list, ListInternal, List);
-
- row_complex_accessor!(get_map, MapInternal, Map);
-}
-
-/// Constructs a `Row` from the list of `fields` and returns it.
-#[inline]
-pub fn make_row(fields: Vec<(String, Field)>) -> Row {
- Row { fields }
-}
-
-impl fmt::Display for Row {
- fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
- write!(f, "{{")?;
- for (i, &(ref key, ref value)) in self.fields.iter().enumerate() {
- key.fmt(f)?;
- write!(f, ": ")?;
- value.fmt(f)?;
- if i < self.fields.len() - 1 {
- write!(f, ", ")?;
- }
- }
- write!(f, "}}")
- }
-}
-
-/// `List` represents a list which contains an array of elements.
-#[derive(Clone, Debug, PartialEq)]
-pub struct List {
- elements: Vec<Field>,
-}
-
-#[allow(clippy::len_without_is_empty)]
-impl List {
- /// Get the number of fields in this row
- pub fn len(&self) -> usize {
- self.elements.len()
- }
-
- pub fn elements(&self) -> &[Field] {
- self.elements.as_slice()
- }
-}
-
-/// Constructs a `List` from the list of `fields` and returns it.
-#[inline]
-pub fn make_list(elements: Vec<Field>) -> List {
- List { elements }
-}
-
-/// Trait for type-safe access of an index for a `List`.
-/// Note that the get_XXX methods do not do bound checking.
-pub trait ListAccessor {
- fn get_bool(&self, i: usize) -> Result<bool>;
- fn get_byte(&self, i: usize) -> Result<i8>;
- fn get_short(&self, i: usize) -> Result<i16>;
- fn get_int(&self, i: usize) -> Result<i32>;
- fn get_long(&self, i: usize) -> Result<i64>;
- fn get_ubyte(&self, i: usize) -> Result<u8>;
- fn get_ushort(&self, i: usize) -> Result<u16>;
- fn get_uint(&self, i: usize) -> Result<u32>;
- fn get_ulong(&self, i: usize) -> Result<u64>;
- fn get_float(&self, i: usize) -> Result<f32>;
- fn get_double(&self, i: usize) -> Result<f64>;
- fn get_timestamp_millis(&self, i: usize) -> Result<u64>;
- fn get_timestamp_micros(&self, i: usize) -> Result<u64>;
- fn get_decimal(&self, i: usize) -> Result<&Decimal>;
- fn get_string(&self, i: usize) -> Result<&String>;
- fn get_bytes(&self, i: usize) -> Result<&ByteArray>;
- fn get_group(&self, i: usize) -> Result<&Row>;
- fn get_list(&self, i: usize) -> Result<&List>;
- fn get_map(&self, i: usize) -> Result<&Map>;
-}
-
-/// Macro to generate type-safe get_xxx methods for primitive types,
-/// e.g. get_bool, get_short
-macro_rules! list_primitive_accessor {
- ($METHOD:ident, $VARIANT:ident, $TY:ty) => {
- fn $METHOD(&self, i: usize) -> Result<$TY> {
- match self.elements[i] {
- Field::$VARIANT(v) => Ok(v),
- _ => Err(general_err!(
- "Cannot access {} as {}",
- self.elements[i].get_type_name(),
- stringify!($VARIANT)
- )),
- }
- }
- };
-}
-
-/// Macro to generate type-safe get_xxx methods for reference types
-/// e.g. get_list, get_map
-macro_rules! list_complex_accessor {
- ($METHOD:ident, $VARIANT:ident, $TY:ty) => {
- fn $METHOD(&self, i: usize) -> Result<&$TY> {
- match self.elements[i] {
- Field::$VARIANT(ref v) => Ok(v),
- _ => Err(general_err!(
- "Cannot access {} as {}",
- self.elements[i].get_type_name(),
- stringify!($VARIANT)
- )),
- }
- }
- };
-}
-
-impl ListAccessor for List {
- list_primitive_accessor!(get_bool, Bool, bool);
-
- list_primitive_accessor!(get_byte, Byte, i8);
-
- list_primitive_accessor!(get_short, Short, i16);
-
- list_primitive_accessor!(get_int, Int, i32);
-
- list_primitive_accessor!(get_long, Long, i64);
-
- list_primitive_accessor!(get_ubyte, UByte, u8);
-
- list_primitive_accessor!(get_ushort, UShort, u16);
-
- list_primitive_accessor!(get_uint, UInt, u32);
-
- list_primitive_accessor!(get_ulong, ULong, u64);
-
- list_primitive_accessor!(get_float, Float, f32);
-
- list_primitive_accessor!(get_double, Double, f64);
-
- list_primitive_accessor!(get_timestamp_millis, TimestampMillis, u64);
-
- list_primitive_accessor!(get_timestamp_micros, TimestampMicros, u64);
-
- list_complex_accessor!(get_decimal, Decimal, Decimal);
-
- list_complex_accessor!(get_string, Str, String);
-
- list_complex_accessor!(get_bytes, Bytes, ByteArray);
-
- list_complex_accessor!(get_group, Group, Row);
-
- list_complex_accessor!(get_list, ListInternal, List);
-
- list_complex_accessor!(get_map, MapInternal, Map);
-}
-
-/// `Map` represents a map which contains a list of key->value pairs.
-#[derive(Clone, Debug, PartialEq)]
-pub struct Map {
- entries: Vec<(Field, Field)>,
-}
-
-#[allow(clippy::len_without_is_empty)]
-impl Map {
- /// Get the number of fields in this row
- pub fn len(&self) -> usize {
- self.entries.len()
- }
-
- pub fn entries(&self) -> &[(Field, Field)] {
- self.entries.as_slice()
- }
-}
-
-/// Constructs a `Map` from the list of `entries` and returns it.
-#[inline]
-pub fn make_map(entries: Vec<(Field, Field)>) -> Map {
- Map { entries }
-}
-
-/// Trait for type-safe access of an index for a `Map`
-pub trait MapAccessor {
- fn get_keys<'a>(&'a self) -> Box<dyn ListAccessor + 'a>;
- fn get_values<'a>(&'a self) -> Box<dyn ListAccessor + 'a>;
-}
-
-struct MapList<'a> {
- elements: Vec<&'a Field>,
-}
-
-/// Macro to generate type-safe get_xxx methods for primitive types,
-/// e.g. get_bool, get_short
-macro_rules! map_list_primitive_accessor {
- ($METHOD:ident, $VARIANT:ident, $TY:ty) => {
- fn $METHOD(&self, i: usize) -> Result<$TY> {
- match self.elements[i] {
- Field::$VARIANT(v) => Ok(*v),
- _ => Err(general_err!(
- "Cannot access {} as {}",
- self.elements[i].get_type_name(),
- stringify!($VARIANT)
- )),
- }
- }
- };
-}
-
-impl<'a> ListAccessor for MapList<'a> {
- map_list_primitive_accessor!(get_bool, Bool, bool);
-
- map_list_primitive_accessor!(get_byte, Byte, i8);
-
- map_list_primitive_accessor!(get_short, Short, i16);
-
- map_list_primitive_accessor!(get_int, Int, i32);
-
- map_list_primitive_accessor!(get_long, Long, i64);
-
- map_list_primitive_accessor!(get_ubyte, UByte, u8);
-
- map_list_primitive_accessor!(get_ushort, UShort, u16);
-
- map_list_primitive_accessor!(get_uint, UInt, u32);
-
- map_list_primitive_accessor!(get_ulong, ULong, u64);
-
- map_list_primitive_accessor!(get_float, Float, f32);
-
- map_list_primitive_accessor!(get_double, Double, f64);
-
- map_list_primitive_accessor!(get_timestamp_millis, TimestampMillis, u64);
-
- map_list_primitive_accessor!(get_timestamp_micros, TimestampMicros, u64);
-
- list_complex_accessor!(get_decimal, Decimal, Decimal);
-
- list_complex_accessor!(get_string, Str, String);
-
- list_complex_accessor!(get_bytes, Bytes, ByteArray);
-
- list_complex_accessor!(get_group, Group, Row);
-
- list_complex_accessor!(get_list, ListInternal, List);
-
- list_complex_accessor!(get_map, MapInternal, Map);
-}
-
-impl MapAccessor for Map {
- fn get_keys<'a>(&'a self) -> Box<dyn ListAccessor + 'a> {
- let map_list = MapList {
- elements: self.entries.iter().map(|v| &v.0).collect(),
- };
- Box::new(map_list)
- }
-
- fn get_values<'a>(&'a self) -> Box<dyn ListAccessor + 'a> {
- let map_list = MapList {
- elements: self.entries.iter().map(|v| &v.1).collect(),
- };
- Box::new(map_list)
- }
-}
-
-/// API to represent a single field in a `Row`.
-#[derive(Clone, Debug, PartialEq)]
-pub enum Field {
- // Primitive types
- /// Null value.
- Null,
- /// Boolean value (`true`, `false`).
- Bool(bool),
- /// Signed integer INT_8.
- Byte(i8),
- /// Signed integer INT_16.
- Short(i16),
- /// Signed integer INT_32.
- Int(i32),
- /// Signed integer INT_64.
- Long(i64),
- // Unsigned integer UINT_8.
- UByte(u8),
- // Unsigned integer UINT_16.
- UShort(u16),
- // Unsigned integer UINT_32.
- UInt(u32),
- // Unsigned integer UINT_64.
- ULong(u64),
- /// IEEE 32-bit floating point value.
- Float(f32),
- /// IEEE 64-bit floating point value.
- Double(f64),
- /// Decimal value.
- Decimal(Decimal),
- /// UTF-8 encoded character string.
- Str(String),
- /// General binary value.
- Bytes(ByteArray),
- /// Date without a time of day, stores the number of days from the
- /// Unix epoch, 1 January 1970.
- Date(u32),
- /// Milliseconds from the Unix epoch, 1 January 1970.
- TimestampMillis(u64),
- /// Microseconds from the Unix epoch, 1 Janiary 1970.
- TimestampMicros(u64),
-
- // ----------------------------------------------------------------------
- // Complex types
- /// Struct, child elements are tuples of field-value pairs.
- Group(Row),
- /// List of elements.
- ListInternal(List),
- /// List of key-value pairs.
- MapInternal(Map),
-}
-
-impl Field {
- /// Get the type name.
- fn get_type_name(&self) -> &'static str {
- match *self {
- Field::Null => "Null",
- Field::Bool(_) => "Bool",
- Field::Byte(_) => "Byte",
- Field::Short(_) => "Short",
- Field::Int(_) => "Int",
- Field::Long(_) => "Long",
- Field::UByte(_) => "UByte",
- Field::UShort(_) => "UShort",
- Field::UInt(_) => "UInt",
- Field::ULong(_) => "ULong",
- Field::Float(_) => "Float",
- Field::Double(_) => "Double",
- Field::Decimal(_) => "Decimal",
- Field::Date(_) => "Date",
- Field::Str(_) => "Str",
- Field::Bytes(_) => "Bytes",
- Field::TimestampMillis(_) => "TimestampMillis",
- Field::TimestampMicros(_) => "TimestampMicros",
- Field::Group(_) => "Group",
- Field::ListInternal(_) => "ListInternal",
- Field::MapInternal(_) => "MapInternal",
- }
- }
-
- /// Determines if this Row represents a primitive value.
- pub fn is_primitive(&self) -> bool {
- !matches!(
- *self,
- Field::Group(_) | Field::ListInternal(_) | Field::MapInternal(_)
- )
- }
-
- /// Converts Parquet BOOLEAN type with logical type into `bool` value.
- #[inline]
- pub fn convert_bool(_descr: &ColumnDescPtr, value: bool) -> Self {
- Field::Bool(value)
- }
-
- /// Converts Parquet INT32 type with converted type into `i32` value.
- #[inline]
- pub fn convert_int32(descr: &ColumnDescPtr, value: i32) -> Self {
- match descr.converted_type() {
- ConvertedType::INT_8 => Field::Byte(value as i8),
- ConvertedType::INT_16 => Field::Short(value as i16),
- ConvertedType::INT_32 | ConvertedType::NONE => Field::Int(value),
- ConvertedType::UINT_8 => Field::UByte(value as u8),
- ConvertedType::UINT_16 => Field::UShort(value as u16),
- ConvertedType::UINT_32 => Field::UInt(value as u32),
- ConvertedType::DATE => Field::Date(value as u32),
- ConvertedType::DECIMAL => Field::Decimal(Decimal::from_i32(
- value,
- descr.type_precision(),
- descr.type_scale(),
- )),
- _ => nyi!(descr, value),
- }
- }
-
- /// Converts Parquet INT64 type with converted type into `i64` value.
- #[inline]
- pub fn convert_int64(descr: &ColumnDescPtr, value: i64) -> Self {
- match descr.converted_type() {
- ConvertedType::INT_64 | ConvertedType::NONE => Field::Long(value),
- ConvertedType::UINT_64 => Field::ULong(value as u64),
- ConvertedType::TIMESTAMP_MILLIS => Field::TimestampMillis(value as u64),
- ConvertedType::TIMESTAMP_MICROS => Field::TimestampMicros(value as u64),
- ConvertedType::DECIMAL => Field::Decimal(Decimal::from_i64(
- value,
- descr.type_precision(),
- descr.type_scale(),
- )),
- _ => nyi!(descr, value),
- }
- }
-
- /// Converts Parquet INT96 (nanosecond timestamps) type and logical type into
- /// `Timestamp` value.
- #[inline]
- pub fn convert_int96(_descr: &ColumnDescPtr, value: Int96) -> Self {
- Field::TimestampMillis(value.to_i64() as u64)
- }
-
- /// Converts Parquet FLOAT type with logical type into `f32` value.
- #[inline]
- pub fn convert_float(_descr: &ColumnDescPtr, value: f32) -> Self {
- Field::Float(value)
- }
-
- /// Converts Parquet DOUBLE type with converted type into `f64` value.
- #[inline]
- pub fn convert_double(_descr: &ColumnDescPtr, value: f64) -> Self {
- Field::Double(value)
- }
-
- /// Converts Parquet BYTE_ARRAY type with converted type into either UTF8 string or
- /// array of bytes.
- #[inline]
- pub fn convert_byte_array(descr: &ColumnDescPtr, value: ByteArray) -> Self {
- match descr.physical_type() {
- PhysicalType::BYTE_ARRAY => match descr.converted_type() {
- ConvertedType::UTF8 | ConvertedType::ENUM | ConvertedType::JSON => {
- let value = String::from_utf8(value.data().to_vec()).unwrap();
- Field::Str(value)
- }
- ConvertedType::BSON | ConvertedType::NONE => Field::Bytes(value),
- ConvertedType::DECIMAL => Field::Decimal(Decimal::from_bytes(
- value,
- descr.type_precision(),
- descr.type_scale(),
- )),
- _ => nyi!(descr, value),
- },
- PhysicalType::FIXED_LEN_BYTE_ARRAY => match descr.converted_type() {
- ConvertedType::DECIMAL => Field::Decimal(Decimal::from_bytes(
- value,
- descr.type_precision(),
- descr.type_scale(),
- )),
- ConvertedType::NONE => Field::Bytes(value),
- _ => nyi!(descr, value),
- },
- _ => nyi!(descr, value),
- }
- }
-
- #[cfg(feature = "cli")]
- pub fn to_json_value(&self) -> Value {
- match &self {
- Field::Null => Value::Null,
- Field::Bool(b) => Value::Bool(*b),
- Field::Byte(n) => Value::Number(serde_json::Number::from(*n)),
- Field::Short(n) => Value::Number(serde_json::Number::from(*n)),
- Field::Int(n) => Value::Number(serde_json::Number::from(*n)),
- Field::Long(n) => Value::Number(serde_json::Number::from(*n)),
- Field::UByte(n) => Value::Number(serde_json::Number::from(*n)),
- Field::UShort(n) => Value::Number(serde_json::Number::from(*n)),
- Field::UInt(n) => Value::Number(serde_json::Number::from(*n)),
- Field::ULong(n) => Value::Number(serde_json::Number::from(*n)),
- Field::Float(n) => serde_json::Number::from_f64(f64::from(*n))
- .map(Value::Number)
- .unwrap_or(Value::Null),
- Field::Double(n) => serde_json::Number::from_f64(*n)
- .map(Value::Number)
- .unwrap_or(Value::Null),
- Field::Decimal(n) => Value::String(convert_decimal_to_string(&n)),
- Field::Str(s) => Value::String(s.to_owned()),
- Field::Bytes(b) => Value::String(base64::encode(b.data())),
- Field::Date(d) => Value::String(convert_date_to_string(*d)),
- Field::TimestampMillis(ts) => {
- Value::String(convert_timestamp_millis_to_string(*ts))
- }
- Field::TimestampMicros(ts) => {
- Value::String(convert_timestamp_micros_to_string(*ts))
- }
- Field::Group(row) => row.to_json_value(),
- Field::ListInternal(fields) => {
- Value::Array(fields.elements.iter().map(|f| f.to_json_value()).collect())
- }
- Field::MapInternal(map) => Value::Object(
- map.entries
- .iter()
- .map(|(key_field, value_field)| {
- let key_val = key_field.to_json_value();
- let key_str = key_val
- .as_str()
- .map(|s| s.to_owned())
- .unwrap_or_else(|| key_val.to_string());
- (key_str, value_field.to_json_value())
- })
- .collect(),
- ),
- }
- }
-}
-
-impl fmt::Display for Field {
- fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
- match *self {
- Field::Null => write!(f, "null"),
- Field::Bool(value) => write!(f, "{}", value),
- Field::Byte(value) => write!(f, "{}", value),
- Field::Short(value) => write!(f, "{}", value),
- Field::Int(value) => write!(f, "{}", value),
- Field::Long(value) => write!(f, "{}", value),
- Field::UByte(value) => write!(f, "{}", value),
- Field::UShort(value) => write!(f, "{}", value),
- Field::UInt(value) => write!(f, "{}", value),
- Field::ULong(value) => write!(f, "{}", value),
- Field::Float(value) => {
- if !(1e-15..=1e19).contains(&value) {
- write!(f, "{:E}", value)
- } else {
- write!(f, "{:?}", value)
- }
- }
- Field::Double(value) => {
- if !(1e-15..=1e19).contains(&value) {
- write!(f, "{:E}", value)
- } else {
- write!(f, "{:?}", value)
- }
- }
- Field::Decimal(ref value) => {
- write!(f, "{}", convert_decimal_to_string(value))
- }
- Field::Str(ref value) => write!(f, "\"{}\"", value),
- Field::Bytes(ref value) => write!(f, "{:?}", value.data()),
- Field::Date(value) => write!(f, "{}", convert_date_to_string(value)),
- Field::TimestampMillis(value) => {
- write!(f, "{}", convert_timestamp_millis_to_string(value))
- }
- Field::TimestampMicros(value) => {
- write!(f, "{}", convert_timestamp_micros_to_string(value))
- }
- Field::Group(ref fields) => write!(f, "{}", fields),
- Field::ListInternal(ref list) => {
- let elems = &list.elements;
- write!(f, "[")?;
- for (i, field) in elems.iter().enumerate() {
- field.fmt(f)?;
- if i < elems.len() - 1 {
- write!(f, ", ")?;
- }
- }
- write!(f, "]")
- }
- Field::MapInternal(ref map) => {
- let entries = &map.entries;
- write!(f, "{{")?;
- for (i, &(ref key, ref value)) in entries.iter().enumerate() {
- key.fmt(f)?;
- write!(f, " -> ")?;
- value.fmt(f)?;
- if i < entries.len() - 1 {
- write!(f, ", ")?;
- }
- }
- write!(f, "}}")
- }
- }
- }
-}
-
-/// Helper method to convert Parquet date into a string.
-/// Input `value` is a number of days since the epoch in UTC.
-/// Date is displayed in local timezone.
-#[inline]
-fn convert_date_to_string(value: u32) -> String {
- static NUM_SECONDS_IN_DAY: i64 = 60 * 60 * 24;
- let dt = Utc.timestamp(value as i64 * NUM_SECONDS_IN_DAY, 0).date();
- format!("{}", dt.format("%Y-%m-%d %:z"))
-}
-
-/// Helper method to convert Parquet timestamp into a string.
-/// Input `value` is a number of milliseconds since the epoch in UTC.
-/// Datetime is displayed in local timezone.
-#[inline]
-fn convert_timestamp_millis_to_string(value: u64) -> String {
- let dt = Utc.timestamp((value / 1000) as i64, 0);
- format!("{}", dt.format("%Y-%m-%d %H:%M:%S %:z"))
-}
-
-/// Helper method to convert Parquet timestamp into a string.
-/// Input `value` is a number of microseconds since the epoch in UTC.
-/// Datetime is displayed in local timezone.
-#[inline]
-fn convert_timestamp_micros_to_string(value: u64) -> String {
- convert_timestamp_millis_to_string(value / 1000)
-}
-
-/// Helper method to convert Parquet decimal into a string.
-/// We assert that `scale >= 0` and `precision > scale`, but this will be enforced
-/// when constructing Parquet schema.
-#[inline]
-fn convert_decimal_to_string(decimal: &Decimal) -> String {
- assert!(decimal.scale() >= 0 && decimal.precision() > decimal.scale());
-
- // Specify as signed bytes to resolve sign as part of conversion.
- let num = BigInt::from_signed_bytes_be(decimal.data());
-
- // Offset of the first digit in a string.
- let negative = if num.sign() == Sign::Minus { 1 } else { 0 };
- let mut num_str = num.to_string();
- let mut point = num_str.len() as i32 - decimal.scale() - negative;
-
- // Convert to string form without scientific notation.
- if point <= 0 {
- // Zeros need to be prepended to the unscaled value.
- while point < 0 {
- num_str.insert(negative as usize, '0');
- point += 1;
- }
- num_str.insert_str(negative as usize, "0.");
- } else {
- // No zeroes need to be prepended to the unscaled value, simply insert decimal
- // point.
- num_str.insert((point + negative) as usize, '.');
- }
-
- num_str
-}
-
-#[cfg(test)]
-#[allow(clippy::approx_constant, clippy::many_single_char_names)]
-mod tests {
- use super::*;
-
- use std::sync::Arc;
-
- use crate::schema::types::{ColumnDescriptor, ColumnPath, PrimitiveTypeBuilder};
-
- /// Creates test column descriptor based on provided type parameters.
- macro_rules! make_column_descr {
- ($physical_type:expr, $logical_type:expr) => {{
- let tpe = PrimitiveTypeBuilder::new("col", $physical_type)
- .with_converted_type($logical_type)
- .build()
- .unwrap();
- Arc::new(ColumnDescriptor::new(
- Arc::new(tpe),
- 0,
- 0,
- ColumnPath::from("col"),
- ))
- }};
- ($physical_type:expr, $logical_type:expr, $len:expr, $prec:expr, $scale:expr) => {{
- let tpe = PrimitiveTypeBuilder::new("col", $physical_type)
- .with_converted_type($logical_type)
- .with_length($len)
- .with_precision($prec)
- .with_scale($scale)
- .build()
- .unwrap();
- Arc::new(ColumnDescriptor::new(
- Arc::new(tpe),
- 0,
- 0,
- ColumnPath::from("col"),
- ))
- }};
- }
-
- #[test]
- fn test_row_convert_bool() {
- // BOOLEAN value does not depend on logical type
- let descr = make_column_descr![PhysicalType::BOOLEAN, ConvertedType::NONE];
-
- let row = Field::convert_bool(&descr, true);
- assert_eq!(row, Field::Bool(true));
-
- let row = Field::convert_bool(&descr, false);
- assert_eq!(row, Field::Bool(false));
- }
-
- #[test]
- fn test_row_convert_int32() {
- let descr = make_column_descr![PhysicalType::INT32, ConvertedType::INT_8];
- let row = Field::convert_int32(&descr, 111);
- assert_eq!(row, Field::Byte(111));
-
- let descr = make_column_descr![PhysicalType::INT32, ConvertedType::INT_16];
- let row = Field::convert_int32(&descr, 222);
- assert_eq!(row, Field::Short(222));
-
- let descr = make_column_descr![PhysicalType::INT32, ConvertedType::INT_32];
- let row = Field::convert_int32(&descr, 333);
- assert_eq!(row, Field::Int(333));
-
- let descr = make_column_descr![PhysicalType::INT32, ConvertedType::UINT_8];
- let row = Field::convert_int32(&descr, -1);
- assert_eq!(row, Field::UByte(255));
-
- let descr = make_column_descr![PhysicalType::INT32, ConvertedType::UINT_16];
- let row = Field::convert_int32(&descr, 256);
- assert_eq!(row, Field::UShort(256));
-
- let descr = make_column_descr![PhysicalType::INT32, ConvertedType::UINT_32];
- let row = Field::convert_int32(&descr, 1234);
- assert_eq!(row, Field::UInt(1234));
-
- let descr = make_column_descr![PhysicalType::INT32, ConvertedType::NONE];
- let row = Field::convert_int32(&descr, 444);
- assert_eq!(row, Field::Int(444));
-
- let descr = make_column_descr![PhysicalType::INT32, ConvertedType::DATE];
- let row = Field::convert_int32(&descr, 14611);
- assert_eq!(row, Field::Date(14611));
-
- let descr =
- make_column_descr![PhysicalType::INT32, ConvertedType::DECIMAL, 0, 8, 2];
- let row = Field::convert_int32(&descr, 444);
- assert_eq!(row, Field::Decimal(Decimal::from_i32(444, 8, 2)));
- }
-
- #[test]
- fn test_row_convert_int64() {
- let descr = make_column_descr![PhysicalType::INT64, ConvertedType::INT_64];
- let row = Field::convert_int64(&descr, 1111);
- assert_eq!(row, Field::Long(1111));
-
- let descr = make_column_descr![PhysicalType::INT64, ConvertedType::UINT_64];
- let row = Field::convert_int64(&descr, 78239823);
- assert_eq!(row, Field::ULong(78239823));
-
- let descr =
- make_column_descr![PhysicalType::INT64, ConvertedType::TIMESTAMP_MILLIS];
- let row = Field::convert_int64(&descr, 1541186529153);
- assert_eq!(row, Field::TimestampMillis(1541186529153));
-
- let descr =
- make_column_descr![PhysicalType::INT64, ConvertedType::TIMESTAMP_MICROS];
- let row = Field::convert_int64(&descr, 1541186529153123);
- assert_eq!(row, Field::TimestampMicros(1541186529153123));
-
- let descr = make_column_descr![PhysicalType::INT64, ConvertedType::NONE];
- let row = Field::convert_int64(&descr, 2222);
- assert_eq!(row, Field::Long(2222));
-
- let descr =
- make_column_descr![PhysicalType::INT64, ConvertedType::DECIMAL, 0, 8, 2];
- let row = Field::convert_int64(&descr, 3333);
- assert_eq!(row, Field::Decimal(Decimal::from_i64(3333, 8, 2)));
- }
-
- #[test]
- fn test_row_convert_int96() {
- // INT96 value does not depend on logical type
- let descr = make_column_descr![PhysicalType::INT96, ConvertedType::NONE];
-
- let value = Int96::from(vec![0, 0, 2454923]);
- let row = Field::convert_int96(&descr, value);
- assert_eq!(row, Field::TimestampMillis(1238544000000));
-
- let value = Int96::from(vec![4165425152, 13, 2454923]);
- let row = Field::convert_int96(&descr, value);
- assert_eq!(row, Field::TimestampMillis(1238544060000));
- }
-
- #[test]
- fn test_row_convert_float() {
- // FLOAT value does not depend on logical type
- let descr = make_column_descr![PhysicalType::FLOAT, ConvertedType::NONE];
- let row = Field::convert_float(&descr, 2.31);
- assert_eq!(row, Field::Float(2.31));
- }
-
- #[test]
- fn test_row_convert_double() {
- // DOUBLE value does not depend on logical type
- let descr = make_column_descr![PhysicalType::DOUBLE, ConvertedType::NONE];
- let row = Field::convert_double(&descr, 1.56);
- assert_eq!(row, Field::Double(1.56));
- }
-
- #[test]
- fn test_row_convert_byte_array() {
- // UTF8
- let descr = make_column_descr![PhysicalType::BYTE_ARRAY, ConvertedType::UTF8];
- let value = ByteArray::from(vec![b'A', b'B', b'C', b'D']);
- let row = Field::convert_byte_array(&descr, value);
- assert_eq!(row, Field::Str("ABCD".to_string()));
-
- // ENUM
- let descr = make_column_descr![PhysicalType::BYTE_ARRAY, ConvertedType::ENUM];
- let value = ByteArray::from(vec![b'1', b'2', b'3']);
- let row = Field::convert_byte_array(&descr, value);
- assert_eq!(row, Field::Str("123".to_string()));
-
- // JSON
- let descr = make_column_descr![PhysicalType::BYTE_ARRAY, ConvertedType::JSON];
- let value = ByteArray::from(vec![b'{', b'"', b'a', b'"', b':', b'1', b'}']);
- let row = Field::convert_byte_array(&descr, value);
- assert_eq!(row, Field::Str("{\"a\":1}".to_string()));
-
- // NONE
- let descr = make_column_descr![PhysicalType::BYTE_ARRAY, ConvertedType::NONE];
- let value = ByteArray::from(vec![1, 2, 3, 4, 5]);
- let row = Field::convert_byte_array(&descr, value.clone());
- assert_eq!(row, Field::Bytes(value));
-
- // BSON
- let descr = make_column_descr![PhysicalType::BYTE_ARRAY, ConvertedType::BSON];
- let value = ByteArray::from(vec![1, 2, 3, 4, 5]);
- let row = Field::convert_byte_array(&descr, value.clone());
- assert_eq!(row, Field::Bytes(value));
-
- // DECIMAL
- let descr =
- make_column_descr![PhysicalType::BYTE_ARRAY, ConvertedType::DECIMAL, 0, 8, 2];
- let value = ByteArray::from(vec![207, 200]);
- let row = Field::convert_byte_array(&descr, value.clone());
- assert_eq!(row, Field::Decimal(Decimal::from_bytes(value, 8, 2)));
-
- // DECIMAL (FIXED_LEN_BYTE_ARRAY)
- let descr = make_column_descr![
- PhysicalType::FIXED_LEN_BYTE_ARRAY,
- ConvertedType::DECIMAL,
- 8,
- 17,
- 5
- ];
- let value = ByteArray::from(vec![0, 0, 0, 0, 0, 4, 147, 224]);
- let row = Field::convert_byte_array(&descr, value.clone());
- assert_eq!(row, Field::Decimal(Decimal::from_bytes(value, 17, 5)));
-
- // NONE (FIXED_LEN_BYTE_ARRAY)
- let descr = make_column_descr![
- PhysicalType::FIXED_LEN_BYTE_ARRAY,
- ConvertedType::NONE,
- 6,
- 0,
- 0
- ];
- let value = ByteArray::from(vec![1, 2, 3, 4, 5, 6]);
- let row = Field::convert_byte_array(&descr, value.clone());
- assert_eq!(row, Field::Bytes(value));
- }
-
- #[test]
- fn test_convert_date_to_string() {
- fn check_date_conversion(y: u32, m: u32, d: u32) {
- let datetime = chrono::NaiveDate::from_ymd(y as i32, m, d).and_hms(0, 0, 0);
- let dt = Utc.from_utc_datetime(&datetime);
- let res = convert_date_to_string((dt.timestamp() / 60 / 60 / 24) as u32);
- let exp = format!("{}", dt.format("%Y-%m-%d %:z"));
- assert_eq!(res, exp);
- }
-
- check_date_conversion(2010, 1, 2);
- check_date_conversion(2014, 5, 1);
- check_date_conversion(2016, 2, 29);
- check_date_conversion(2017, 9, 12);
- check_date_conversion(2018, 3, 31);
- }
-
- #[test]
- fn test_convert_timestamp_to_string() {
- fn check_datetime_conversion(y: u32, m: u32, d: u32, h: u32, mi: u32, s: u32) {
- let datetime = chrono::NaiveDate::from_ymd(y as i32, m, d).and_hms(h, mi, s);
- let dt = Utc.from_utc_datetime(&datetime);
- let res = convert_timestamp_millis_to_string(dt.timestamp_millis() as u64);
- let exp = format!("{}", dt.format("%Y-%m-%d %H:%M:%S %:z"));
- assert_eq!(res, exp);
- }
-
- check_datetime_conversion(2010, 1, 2, 13, 12, 54);
- check_datetime_conversion(2011, 1, 3, 8, 23, 1);
- check_datetime_conversion(2012, 4, 5, 11, 6, 32);
- check_datetime_conversion(2013, 5, 12, 16, 38, 0);
- check_datetime_conversion(2014, 11, 28, 21, 15, 12);
- }
-
- #[test]
- fn test_convert_float_to_string() {
- assert_eq!(format!("{}", Field::Float(1.0)), "1.0");
- assert_eq!(format!("{}", Field::Float(9.63)), "9.63");
- assert_eq!(format!("{}", Field::Float(1e-15)), "0.000000000000001");
- assert_eq!(format!("{}", Field::Float(1e-16)), "1E-16");
- assert_eq!(format!("{}", Field::Float(1e19)), "10000000000000000000.0");
- assert_eq!(format!("{}", Field::Float(1e20)), "1E20");
- assert_eq!(format!("{}", Field::Float(1.7976931E30)), "1.7976931E30");
- assert_eq!(format!("{}", Field::Float(-1.7976931E30)), "-1.7976931E30");
- }
-
- #[test]
- fn test_convert_double_to_string() {
- assert_eq!(format!("{}", Field::Double(1.0)), "1.0");
- assert_eq!(format!("{}", Field::Double(9.63)), "9.63");
- assert_eq!(format!("{}", Field::Double(1e-15)), "0.000000000000001");
- assert_eq!(format!("{}", Field::Double(1e-16)), "1E-16");
- assert_eq!(format!("{}", Field::Double(1e19)), "10000000000000000000.0");
- assert_eq!(format!("{}", Field::Double(1e20)), "1E20");
- assert_eq!(
- format!("{}", Field::Double(1.79769313486E308)),
- "1.79769313486E308"
- );
- assert_eq!(
- format!("{}", Field::Double(-1.79769313486E308)),
- "-1.79769313486E308"
- );
- }
-
- #[test]
- fn test_convert_decimal_to_string() {
- // Helper method to compare decimal
- fn check_decimal(bytes: Vec<u8>, precision: i32, scale: i32, res: &str) {
- let decimal = Decimal::from_bytes(ByteArray::from(bytes), precision, scale);
- assert_eq!(convert_decimal_to_string(&decimal), res);
- }
-
- // This example previously used to fail in some engines
- check_decimal(
- vec![0, 0, 0, 0, 0, 0, 0, 0, 13, 224, 182, 179, 167, 100, 0, 0],
- 38,
- 18,
- "1.000000000000000000",
- );
- check_decimal(
- vec![
- 249, 233, 247, 16, 185, 192, 202, 223, 215, 165, 192, 166, 67, 72,
- ],
- 36,
- 28,
- "-12344.0242342304923409234234293432",
- );
- check_decimal(vec![0, 0, 0, 0, 0, 4, 147, 224], 17, 5, "3.00000");
- check_decimal(vec![0, 0, 0, 0, 1, 201, 195, 140], 18, 2, "300000.12");
- check_decimal(vec![207, 200], 10, 2, "-123.44");
- check_decimal(vec![207, 200], 10, 8, "-0.00012344");
- }
-
- #[test]
- fn test_row_display() {
- // Primitive types
- assert_eq!(format!("{}", Field::Null), "null");
- assert_eq!(format!("{}", Field::Bool(true)), "true");
- assert_eq!(format!("{}", Field::Bool(false)), "false");
- assert_eq!(format!("{}", Field::Byte(1)), "1");
- assert_eq!(format!("{}", Field::Short(2)), "2");
- assert_eq!(format!("{}", Field::Int(3)), "3");
- assert_eq!(format!("{}", Field::Long(4)), "4");
- assert_eq!(format!("{}", Field::UByte(1)), "1");
- assert_eq!(format!("{}", Field::UShort(2)), "2");
- assert_eq!(format!("{}", Field::UInt(3)), "3");
- assert_eq!(format!("{}", Field::ULong(4)), "4");
- assert_eq!(format!("{}", Field::Float(5.0)), "5.0");
- assert_eq!(format!("{}", Field::Float(5.1234)), "5.1234");
- assert_eq!(format!("{}", Field::Double(6.0)), "6.0");
- assert_eq!(format!("{}", Field::Double(6.1234)), "6.1234");
- assert_eq!(format!("{}", Field::Str("abc".to_string())), "\"abc\"");
- assert_eq!(
- format!("{}", Field::Bytes(ByteArray::from(vec![1, 2, 3]))),
- "[1, 2, 3]"
- );
- assert_eq!(
- format!("{}", Field::Date(14611)),
- convert_date_to_string(14611)
- );
- assert_eq!(
- format!("{}", Field::TimestampMillis(1262391174000)),
- convert_timestamp_millis_to_string(1262391174000)
- );
- assert_eq!(
- format!("{}", Field::TimestampMicros(1262391174000000)),
- convert_timestamp_micros_to_string(1262391174000000)
- );
- assert_eq!(
- format!("{}", Field::Decimal(Decimal::from_i32(4, 8, 2))),
- convert_decimal_to_string(&Decimal::from_i32(4, 8, 2))
- );
-
- // Complex types
- let fields = vec![
- ("x".to_string(), Field::Null),
- ("Y".to_string(), Field::Int(2)),
- ("z".to_string(), Field::Float(3.1)),
- ("a".to_string(), Field::Str("abc".to_string())),
- ];
- let row = Field::Group(make_row(fields));
- assert_eq!(format!("{}", row), "{x: null, Y: 2, z: 3.1, a: \"abc\"}");
-
- let row = Field::ListInternal(make_list(vec![
- Field::Int(2),
- Field::Int(1),
- Field::Null,
- Field::Int(12),
- ]));
- assert_eq!(format!("{}", row), "[2, 1, null, 12]");
-
- let row = Field::MapInternal(make_map(vec![
- (Field::Int(1), Field::Float(1.2)),
- (Field::Int(2), Field::Float(4.5)),
- (Field::Int(3), Field::Float(2.3)),
- ]));
- assert_eq!(format!("{}", row), "{1 -> 1.2, 2 -> 4.5, 3 -> 2.3}");
- }
-
- #[test]
- fn test_is_primitive() {
- // primitives
- assert!(Field::Null.is_primitive());
- assert!(Field::Bool(true).is_primitive());
- assert!(Field::Bool(false).is_primitive());
- assert!(Field::Byte(1).is_primitive());
- assert!(Field::Short(2).is_primitive());
- assert!(Field::Int(3).is_primitive());
- assert!(Field::Long(4).is_primitive());
- assert!(Field::UByte(1).is_primitive());
- assert!(Field::UShort(2).is_primitive());
- assert!(Field::UInt(3).is_primitive());
- assert!(Field::ULong(4).is_primitive());
- assert!(Field::Float(5.0).is_primitive());
- assert!(Field::Float(5.1234).is_primitive());
- assert!(Field::Double(6.0).is_primitive());
- assert!(Field::Double(6.1234).is_primitive());
- assert!(Field::Str("abc".to_string()).is_primitive());
- assert!(Field::Bytes(ByteArray::from(vec![1, 2, 3])).is_primitive());
- assert!(Field::TimestampMillis(12345678).is_primitive());
- assert!(Field::TimestampMicros(12345678901).is_primitive());
- assert!(Field::Decimal(Decimal::from_i32(4, 8, 2)).is_primitive());
-
- // complex types
- assert_eq!(
- false,
- Field::Group(make_row(vec![
- ("x".to_string(), Field::Null),
- ("Y".to_string(), Field::Int(2)),
- ("z".to_string(), Field::Float(3.1)),
- ("a".to_string(), Field::Str("abc".to_string()))
- ]))
- .is_primitive()
- );
-
- assert_eq!(
- false,
- Field::ListInternal(make_list(vec![
- Field::Int(2),
- Field::Int(1),
- Field::Null,
- Field::Int(12)
- ]))
- .is_primitive()
- );
-
- assert_eq!(
- false,
- Field::MapInternal(make_map(vec![
- (Field::Int(1), Field::Float(1.2)),
- (Field::Int(2), Field::Float(4.5)),
- (Field::Int(3), Field::Float(2.3))
- ]))
- .is_primitive()
- );
- }
-
- #[test]
- fn test_row_primitive_field_fmt() {
- // Primitives types
- let row = make_row(vec![
- ("00".to_string(), Field::Null),
- ("01".to_string(), Field::Bool(false)),
- ("02".to_string(), Field::Byte(3)),
- ("03".to_string(), Field::Short(4)),
- ("04".to_string(), Field::Int(5)),
- ("05".to_string(), Field::Long(6)),
- ("06".to_string(), Field::UByte(7)),
- ("07".to_string(), Field::UShort(8)),
- ("08".to_string(), Field::UInt(9)),
- ("09".to_string(), Field::ULong(10)),
- ("10".to_string(), Field::Float(11.1)),
- ("11".to_string(), Field::Double(12.1)),
- ("12".to_string(), Field::Str("abc".to_string())),
- (
- "13".to_string(),
- Field::Bytes(ByteArray::from(vec![1, 2, 3, 4, 5])),
- ),
- ("14".to_string(), Field::Date(14611)),
- ("15".to_string(), Field::TimestampMillis(1262391174000)),
- ("16".to_string(), Field::TimestampMicros(1262391174000000)),
- ("17".to_string(), Field::Decimal(Decimal::from_i32(4, 7, 2))),
- ]);
-
- assert_eq!("null", format!("{}", row.fmt(0)));
- assert_eq!("false", format!("{}", row.fmt(1)));
- assert_eq!("3", format!("{}", row.fmt(2)));
- assert_eq!("4", format!("{}", row.fmt(3)));
- assert_eq!("5", format!("{}", row.fmt(4)));
- assert_eq!("6", format!("{}", row.fmt(5)));
- assert_eq!("7", format!("{}", row.fmt(6)));
- assert_eq!("8", format!("{}", row.fmt(7)));
- assert_eq!("9", format!("{}", row.fmt(8)));
- assert_eq!("10", format!("{}", row.fmt(9)));
- assert_eq!("11.1", format!("{}", row.fmt(10)));
- assert_eq!("12.1", format!("{}", row.fmt(11)));
- assert_eq!("\"abc\"", format!("{}", row.fmt(12)));
- assert_eq!("[1, 2, 3, 4, 5]", format!("{}", row.fmt(13)));
- assert_eq!(convert_date_to_string(14611), format!("{}", row.fmt(14)));
- assert_eq!(
- convert_timestamp_millis_to_string(1262391174000),
- format!("{}", row.fmt(15))
- );
- assert_eq!(
- convert_timestamp_micros_to_string(1262391174000000),
- format!("{}", row.fmt(16))
- );
- assert_eq!("0.04", format!("{}", row.fmt(17)));
- }
-
- #[test]
- fn test_row_complex_field_fmt() {
- // Complex types
- let row = make_row(vec![
- (
- "00".to_string(),
- Field::Group(make_row(vec![
- ("x".to_string(), Field::Null),
- ("Y".to_string(), Field::Int(2)),
- ])),
- ),
- (
- "01".to_string(),
- Field::ListInternal(make_list(vec![
- Field::Int(2),
- Field::Int(1),
- Field::Null,
- Field::Int(12),
- ])),
- ),
- (
- "02".to_string(),
- Field::MapInternal(make_map(vec![
- (Field::Int(1), Field::Float(1.2)),
- (Field::Int(2), Field::Float(4.5)),
- (Field::Int(3), Field::Float(2.3)),
- ])),
- ),
- ]);
-
- assert_eq!("{x: null, Y: 2}", format!("{}", row.fmt(0)));
- assert_eq!("[2, 1, null, 12]", format!("{}", row.fmt(1)));
- assert_eq!("{1 -> 1.2, 2 -> 4.5, 3 -> 2.3}", format!("{}", row.fmt(2)));
- }
-
- #[test]
- fn test_row_primitive_accessors() {
- // primitives
- let row = make_row(vec![
- ("a".to_string(), Field::Null),
- ("b".to_string(), Field::Bool(false)),
- ("c".to_string(), Field::Byte(3)),
- ("d".to_string(), Field::Short(4)),
- ("e".to_string(), Field::Int(5)),
- ("f".to_string(), Field::Long(6)),
- ("g".to_string(), Field::UByte(3)),
- ("h".to_string(), Field::UShort(4)),
- ("i".to_string(), Field::UInt(5)),
- ("j".to_string(), Field::ULong(6)),
- ("k".to_string(), Field::Float(7.1)),
- ("l".to_string(), Field::Double(8.1)),
- ("m".to_string(), Field::Str("abc".to_string())),
- (
- "n".to_string(),
- Field::Bytes(ByteArray::from(vec![1, 2, 3, 4, 5])),
- ),
- ("o".to_string(), Field::Decimal(Decimal::from_i32(4, 7, 2))),
- ]);
-
- assert_eq!(false, row.get_bool(1).unwrap());
- assert_eq!(3, row.get_byte(2).unwrap());
- assert_eq!(4, row.get_short(3).unwrap());
- assert_eq!(5, row.get_int(4).unwrap());
- assert_eq!(6, row.get_long(5).unwrap());
- assert_eq!(3, row.get_ubyte(6).unwrap());
- assert_eq!(4, row.get_ushort(7).unwrap());
- assert_eq!(5, row.get_uint(8).unwrap());
- assert_eq!(6, row.get_ulong(9).unwrap());
- assert!(7.1 - row.get_float(10).unwrap() < f32::EPSILON);
- assert!(8.1 - row.get_double(11).unwrap() < f64::EPSILON);
- assert_eq!("abc", row.get_string(12).unwrap());
- assert_eq!(5, row.get_bytes(13).unwrap().len());
- assert_eq!(7, row.get_decimal(14).unwrap().precision());
- }
-
- #[test]
- fn test_row_primitive_invalid_accessors() {
- // primitives
- let row = make_row(vec![
- ("a".to_string(), Field::Null),
- ("b".to_string(), Field::Bool(false)),
- ("c".to_string(), Field::Byte(3)),
- ("d".to_string(), Field::Short(4)),
- ("e".to_string(), Field::Int(5)),
- ("f".to_string(), Field::Long(6)),
- ("g".to_string(), Field::UByte(3)),
- ("h".to_string(), Field::UShort(4)),
- ("i".to_string(), Field::UInt(5)),
- ("j".to_string(), Field::ULong(6)),
- ("k".to_string(), Field::Float(7.1)),
- ("l".to_string(), Field::Double(8.1)),
- ("m".to_string(), Field::Str("abc".to_string())),
- (
- "n".to_string(),
- Field::Bytes(ByteArray::from(vec![1, 2, 3, 4, 5])),
- ),
- ("o".to_string(), Field::Decimal(Decimal::from_i32(4, 7, 2))),
- ]);
-
- for i in 0..row.len() {
- assert!(row.get_group(i).is_err());
- }
- }
-
- #[test]
- fn test_row_complex_accessors() {
- let row = make_row(vec![
- (
- "a".to_string(),
- Field::Group(make_row(vec![
- ("x".to_string(), Field::Null),
- ("Y".to_string(), Field::Int(2)),
- ])),
- ),
- (
- "b".to_string(),
- Field::ListInternal(make_list(vec![
- Field::Int(2),
- Field::Int(1),
- Field::Null,
- Field::Int(12),
- ])),
- ),
- (
- "c".to_string(),
- Field::MapInternal(make_map(vec![
- (Field::Int(1), Field::Float(1.2)),
- (Field::Int(2), Field::Float(4.5)),
- (Field::Int(3), Field::Float(2.3)),
- ])),
- ),
- ]);
-
- assert_eq!(2, row.get_group(0).unwrap().len());
- assert_eq!(4, row.get_list(1).unwrap().len());
- assert_eq!(3, row.get_map(2).unwrap().len());
- }
-
- #[test]
- fn test_row_complex_invalid_accessors() {
- let row = make_row(vec![
- (
- "a".to_string(),
- Field::Group(make_row(vec![
- ("x".to_string(), Field::Null),
- ("Y".to_string(), Field::Int(2)),
- ])),
- ),
- (
- "b".to_string(),
- Field::ListInternal(make_list(vec![
- Field::Int(2),
- Field::Int(1),
- Field::Null,
- Field::Int(12),
- ])),
- ),
- (
- "c".to_string(),
- Field::MapInternal(make_map(vec![
- (Field::Int(1), Field::Float(1.2)),
- (Field::Int(2), Field::Float(4.5)),
- (Field::Int(3), Field::Float(2.3)),
- ])),
- ),
- ]);
-
- assert_eq!(
- ParquetError::General("Cannot access Group as Float".to_string()),
- row.get_float(0).unwrap_err()
- );
- assert_eq!(
- ParquetError::General("Cannot access ListInternal as Float".to_string()),
- row.get_float(1).unwrap_err()
- );
- assert_eq!(
- ParquetError::General("Cannot access MapInternal as Float".to_string()),
- row.get_float(2).unwrap_err()
- );
- }
-
- #[test]
- fn test_list_primitive_accessors() {
- // primitives
- let list = make_list(vec![Field::Bool(false)]);
- assert_eq!(false, list.get_bool(0).unwrap());
-
- let list = make_list(vec![Field::Byte(3), Field::Byte(4)]);
- assert_eq!(4, list.get_byte(1).unwrap());
-
- let list = make_list(vec![Field::Short(4), Field::Short(5), Field::Short(6)]);
- assert_eq!(6, list.get_short(2).unwrap());
-
- let list = make_list(vec![Field::Int(5)]);
- assert_eq!(5, list.get_int(0).unwrap());
-
- let list = make_list(vec![Field::Long(6), Field::Long(7)]);
- assert_eq!(7, list.get_long(1).unwrap());
-
- let list = make_list(vec![Field::UByte(3), Field::UByte(4)]);
- assert_eq!(4, list.get_ubyte(1).unwrap());
-
- let list = make_list(vec![Field::UShort(4), Field::UShort(5), Field::UShort(6)]);
- assert_eq!(6, list.get_ushort(2).unwrap());
-
- let list = make_list(vec![Field::UInt(5)]);
- assert_eq!(5, list.get_uint(0).unwrap());
-
- let list = make_list(vec![Field::ULong(6), Field::ULong(7)]);
- assert_eq!(7, list.get_ulong(1).unwrap());
-
- let list = make_list(vec![
- Field::Float(8.1),
- Field::Float(9.2),
- Field::Float(10.3),
- ]);
- assert!(10.3 - list.get_float(2).unwrap() < f32::EPSILON);
-
- let list = make_list(vec![Field::Double(3.1415)]);
- assert!(3.1415 - list.get_double(0).unwrap() < f64::EPSILON);
-
- let list = make_list(vec![Field::Str("abc".to_string())]);
- assert_eq!(&"abc".to_string(), list.get_string(0).unwrap());
-
- let list = make_list(vec![Field::Bytes(ByteArray::from(vec![1, 2, 3, 4, 5]))]);
- assert_eq!(&[1, 2, 3, 4, 5], list.get_bytes(0).unwrap().data());
-
- let list = make_list(vec![Field::Decimal(Decimal::from_i32(4, 5, 2))]);
- assert_eq!(&[0, 0, 0, 4], list.get_decimal(0).unwrap().data());
- }
-
- #[test]
- fn test_list_primitive_invalid_accessors() {
- // primitives
- let list = make_list(vec![Field::Bool(false)]);
- assert!(list.get_byte(0).is_err());
-
- let list = make_list(vec![Field::Byte(3), Field::Byte(4)]);
- assert!(list.get_short(1).is_err());
-
- let list = make_list(vec![Field::Short(4), Field::Short(5), Field::Short(6)]);
- assert!(list.get_int(2).is_err());
-
- let list = make_list(vec![Field::Int(5)]);
- assert!(list.get_long(0).is_err());
-
- let list = make_list(vec![Field::Long(6), Field::Long(7)]);
- assert!(list.get_float(1).is_err());
-
- let list = make_list(vec![Field::UByte(3), Field::UByte(4)]);
- assert!(list.get_short(1).is_err());
-
- let list = make_list(vec![Field::UShort(4), Field::UShort(5), Field::UShort(6)]);
- assert!(list.get_int(2).is_err());
-
- let list = make_list(vec![Field::UInt(5)]);
- assert!(list.get_long(0).is_err());
-
- let list = make_list(vec![Field::ULong(6), Field::ULong(7)]);
- assert!(list.get_float(1).is_err());
-
- let list = make_list(vec![
- Field::Float(8.1),
- Field::Float(9.2),
- Field::Float(10.3),
- ]);
- assert!(list.get_double(2).is_err());
-
- let list = make_list(vec![Field::Double(3.1415)]);
- assert!(list.get_string(0).is_err());
-
- let list = make_list(vec![Field::Str("abc".to_string())]);
- assert!(list.get_bytes(0).is_err());
-
- let list = make_list(vec![Field::Bytes(ByteArray::from(vec![1, 2, 3, 4, 5]))]);
- assert!(list.get_bool(0).is_err());
-
- let list = make_list(vec![Field::Decimal(Decimal::from_i32(4, 5, 2))]);
- assert!(list.get_bool(0).is_err());
- }
-
- #[test]
- fn test_list_complex_accessors() {
- let list = make_list(vec![Field::Group(make_row(vec![
- ("x".to_string(), Field::Null),
- ("Y".to_string(), Field::Int(2)),
- ]))]);
- assert_eq!(2, list.get_group(0).unwrap().len());
-
- let list = make_list(vec![Field::ListInternal(make_list(vec![
- Field::Int(2),
- Field::Int(1),
- Field::Null,
- Field::Int(12),
- ]))]);
- assert_eq!(4, list.get_list(0).unwrap().len());
-
- let list = make_list(vec![Field::MapInternal(make_map(vec![
- (Field::Int(1), Field::Float(1.2)),
- (Field::Int(2), Field::Float(4.5)),
- (Field::Int(3), Field::Float(2.3)),
- ]))]);
- assert_eq!(3, list.get_map(0).unwrap().len());
- }
-
- #[test]
- fn test_list_complex_invalid_accessors() {
- let list = make_list(vec![Field::Group(make_row(vec![
- ("x".to_string(), Field::Null),
- ("Y".to_string(), Field::Int(2)),
- ]))]);
- assert_eq!(
- general_err!("Cannot access Group as Float".to_string()),
- list.get_float(0).unwrap_err()
- );
-
- let list = make_list(vec![Field::ListInternal(make_list(vec![
- Field::Int(2),
- Field::Int(1),
- Field::Null,
- Field::Int(12),
- ]))]);
- assert_eq!(
- general_err!("Cannot access ListInternal as Float".to_string()),
- list.get_float(0).unwrap_err()
- );
-
- let list = make_list(vec![Field::MapInternal(make_map(vec![
- (Field::Int(1), Field::Float(1.2)),
- (Field::Int(2), Field::Float(4.5)),
- (Field::Int(3), Field::Float(2.3)),
- ]))]);
- assert_eq!(
- general_err!("Cannot access MapInternal as Float".to_string()),
- list.get_float(0).unwrap_err()
- );
- }
-
- #[test]
- fn test_map_accessors() {
- // a map from int to string
- let map = make_map(vec![
- (Field::Int(1), Field::Str("a".to_string())),
- (Field::Int(2), Field::Str("b".to_string())),
- (Field::Int(3), Field::Str("c".to_string())),
- (Field::Int(4), Field::Str("d".to_string())),
- (Field::Int(5), Field::Str("e".to_string())),
- ]);
-
- assert_eq!(5, map.len());
- for i in 0..5 {
- assert_eq!((i + 1) as i32, map.get_keys().get_int(i).unwrap());
- assert_eq!(
- &((i as u8 + b'a') as char).to_string(),
- map.get_values().get_string(i).unwrap()
- );
- }
- }
-
- #[test]
- #[cfg(feature = "cli")]
- fn test_to_json_value() {
- assert_eq!(Field::Null.to_json_value(), Value::Null);
- assert_eq!(Field::Bool(true).to_json_value(), Value::Bool(true));
- assert_eq!(Field::Bool(false).to_json_value(), Value::Bool(false));
- assert_eq!(
- Field::Byte(1).to_json_value(),
- Value::Number(serde_json::Number::from(1))
- );
- assert_eq!(
- Field::Short(2).to_json_value(),
- Value::Number(serde_json::Number::from(2))
- );
- assert_eq!(
- Field::Int(3).to_json_value(),
- Value::Number(serde_json::Number::from(3))
- );
- assert_eq!(
- Field::Long(4).to_json_value(),
- Value::Number(serde_json::Number::from(4))
- );
- assert_eq!(
- Field::UByte(1).to_json_value(),
- Value::Number(serde_json::Number::from(1))
- );
- assert_eq!(
- Field::UShort(2).to_json_value(),
- Value::Number(serde_json::Number::from(2))
- );
- assert_eq!(
- Field::UInt(3).to_json_value(),
- Value::Number(serde_json::Number::from(3))
- );
- assert_eq!(
- Field::ULong(4).to_json_value(),
- Value::Number(serde_json::Number::from(4))
- );
- assert_eq!(
- Field::Float(5.0).to_json_value(),
- Value::Number(serde_json::Number::from_f64(f64::from(5.0 as f32)).unwrap())
- );
- assert_eq!(
- Field::Float(5.1234).to_json_value(),
- Value::Number(
- serde_json::Number::from_f64(f64::from(5.1234 as f32)).unwrap()
- )
- );
- assert_eq!(
- Field::Double(6.0).to_json_value(),
- Value::Number(serde_json::Number::from_f64(6.0 as f64).unwrap())
- );
- assert_eq!(
- Field::Double(6.1234).to_json_value(),
- Value::Number(serde_json::Number::from_f64(6.1234 as f64).unwrap())
- );
- assert_eq!(
- Field::Str("abc".to_string()).to_json_value(),
- Value::String(String::from("abc"))
- );
- assert_eq!(
- Field::Decimal(Decimal::from_i32(4, 8, 2)).to_json_value(),
- Value::String(String::from("0.04"))
- );
- assert_eq!(
- Field::Bytes(ByteArray::from(vec![1, 2, 3])).to_json_value(),
- Value::String(String::from("AQID"))
- );
- assert_eq!(
- Field::TimestampMillis(12345678).to_json_value(),
- Value::String("1970-01-01 03:25:45 +00:00".to_string())
- );
- assert_eq!(
- Field::TimestampMicros(12345678901).to_json_value(),
- Value::String(convert_timestamp_micros_to_string(12345678901))
- );
-
- let fields = vec![
- ("X".to_string(), Field::Int(1)),
- ("Y".to_string(), Field::Double(2.2)),
- ("Z".to_string(), Field::Str("abc".to_string())),
- ];
- let row = Field::Group(make_row(fields));
- assert_eq!(
- row.to_json_value(),
- serde_json::json!({"X": 1, "Y": 2.2, "Z": "abc"})
- );
-
- let row = Field::ListInternal(make_list(vec![
- Field::Int(1),
- Field::Int(12),
- Field::Null,
- ]));
- let array = vec![
- Value::Number(serde_json::Number::from(1)),
- Value::Number(serde_json::Number::from(12)),
- Value::Null,
- ];
- assert_eq!(row.to_json_value(), Value::Array(array));
-
- let row = Field::MapInternal(make_map(vec![
- (Field::Str("k1".to_string()), Field::Double(1.2)),
- (Field::Str("k2".to_string()), Field::Double(3.4)),
- (Field::Str("k3".to_string()), Field::Double(4.5)),
- ]));
- assert_eq!(
- row.to_json_value(),
- serde_json::json!({"k1": 1.2, "k2": 3.4, "k3": 4.5})
- );
- }
-}
-
-#[cfg(test)]
-#[allow(clippy::approx_constant, clippy::many_single_char_names)]
-mod api_tests {
- use super::{make_list, make_map, make_row};
- use crate::record::Field;
-
- #[test]
- fn test_field_visibility() {
- let row = make_row(vec![(
- "a".to_string(),
- Field::Group(make_row(vec![
- ("x".to_string(), Field::Null),
- ("Y".to_string(), Field::Int(2)),
- ])),
- )]);
-
- match row.get_column_iter().next() {
- Some(column) => {
- assert_eq!("a", column.0);
- match column.1 {
- Field::Group(r) => {
- assert_eq!(
- &make_row(vec![
- ("x".to_string(), Field::Null),
- ("Y".to_string(), Field::Int(2)),
- ]),
- r
- );
- }
- _ => panic!("Expected the first column to be Field::Group"),
- }
- }
- None => panic!("Expected at least one column"),
- }
- }
-
- #[test]
- fn test_list_element_access() {
- let expected = vec![
- Field::Int(1),
- Field::Group(make_row(vec![
- ("x".to_string(), Field::Null),
- ("Y".to_string(), Field::Int(2)),
- ])),
- ];
-
- let list = make_list(expected.clone());
- assert_eq!(expected.as_slice(), list.elements());
- }
-
- #[test]
- fn test_map_entry_access() {
- let expected = vec![
- (Field::Str("one".to_owned()), Field::Int(1)),
- (Field::Str("two".to_owned()), Field::Int(2)),
- ];
-
- let map = make_map(expected.clone());
- assert_eq!(expected.as_slice(), map.entries());
- }
-}
diff --git a/parquet/src/record/mod.rs b/parquet/src/record/mod.rs
deleted file mode 100644
index fb4abb5..0000000
--- a/parquet/src/record/mod.rs
+++ /dev/null
@@ -1,28 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-//! Contains record-based API for reading Parquet files.
-
-mod api;
-pub mod reader;
-mod record_writer;
-mod triplet;
-
-pub use self::{
- api::{Field, List, ListAccessor, Map, MapAccessor, Row, RowAccessor},
- record_writer::RecordWriter,
-};
diff --git a/parquet/src/record/reader.rs b/parquet/src/record/reader.rs
deleted file mode 100644
index 691afe8..0000000
--- a/parquet/src/record/reader.rs
+++ /dev/null
@@ -1,1667 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-//! Contains implementation of record assembly and converting Parquet types into
-//! [`Row`](crate::record::Row)s.
-
-use std::{collections::HashMap, fmt, sync::Arc};
-
-use crate::basic::{ConvertedType, Repetition};
-use crate::errors::{ParquetError, Result};
-use crate::file::reader::{FileReader, RowGroupReader};
-use crate::record::{
- api::{make_list, make_map, make_row, Field, Row},
- triplet::TripletIter,
-};
-use crate::schema::types::{ColumnPath, SchemaDescPtr, SchemaDescriptor, Type, TypePtr};
-
-/// Default batch size for a reader
-const DEFAULT_BATCH_SIZE: usize = 1024;
-
-/// Tree builder for `Reader` enum.
-/// Serves as a container of options for building a reader tree and a builder, and
-/// accessing a records iterator [`RowIter`].
-pub struct TreeBuilder {
- // Batch size (>= 1) for triplet iterators
- batch_size: usize,
-}
-
-impl TreeBuilder {
- /// Creates new tree builder with default parameters.
- pub fn new() -> Self {
- Self {
- batch_size: DEFAULT_BATCH_SIZE,
- }
- }
-
- /// Sets batch size for this tree builder.
- pub fn with_batch_size(mut self, batch_size: usize) -> Self {
- self.batch_size = batch_size;
- self
- }
-
- /// Creates new root reader for provided schema and row group.
- pub fn build(
- &self,
- descr: SchemaDescPtr,
- row_group_reader: &dyn RowGroupReader,
- ) -> Reader {
- // Prepare lookup table of column path -> original column index
- // This allows to prune columns and map schema leaf nodes to the column readers
- let mut paths: HashMap<ColumnPath, usize> = HashMap::new();
- let row_group_metadata = row_group_reader.metadata();
-
- for col_index in 0..row_group_reader.num_columns() {
- let col_meta = row_group_metadata.column(col_index);
- let col_path = col_meta.column_path().clone();
- paths.insert(col_path, col_index);
- }
-
- // Build child readers for the message type
- let mut readers = Vec::new();
- let mut path = Vec::new();
-
- for field in descr.root_schema().get_fields() {
- let reader = self.reader_tree(
- field.clone(),
- &mut path,
- 0,
- 0,
- &paths,
- row_group_reader,
- );
- readers.push(reader);
- }
-
- // Return group reader for message type,
- // it is always required with definition level 0
- Reader::GroupReader(None, 0, readers)
- }
-
- /// Creates iterator of `Row`s directly from schema descriptor and row group.
- pub fn as_iter(
- &self,
- descr: SchemaDescPtr,
- row_group_reader: &dyn RowGroupReader,
- ) -> ReaderIter {
- let num_records = row_group_reader.metadata().num_rows() as usize;
- ReaderIter::new(self.build(descr, row_group_reader), num_records)
- }
-
- /// Builds tree of readers for the current schema recursively.
- fn reader_tree(
- &self,
- field: TypePtr,
- mut path: &mut Vec<String>,
- mut curr_def_level: i16,
- mut curr_rep_level: i16,
- paths: &HashMap<ColumnPath, usize>,
- row_group_reader: &dyn RowGroupReader,
- ) -> Reader {
- assert!(field.get_basic_info().has_repetition());
- // Update current definition and repetition levels for this type
- let repetition = field.get_basic_info().repetition();
- match repetition {
- Repetition::OPTIONAL => {
- curr_def_level += 1;
- }
- Repetition::REPEATED => {
- curr_def_level += 1;
- curr_rep_level += 1;
- }
- _ => {}
- }
-
- path.push(String::from(field.name()));
- let reader = if field.is_primitive() {
- let col_path = ColumnPath::new(path.to_vec());
- let orig_index = *paths.get(&col_path).unwrap();
- let col_descr = row_group_reader
- .metadata()
- .column(orig_index)
- .column_descr_ptr();
- let col_reader = row_group_reader.get_column_reader(orig_index).unwrap();
- let column = TripletIter::new(col_descr, col_reader, self.batch_size);
- Reader::PrimitiveReader(field, column)
- } else {
- match field.get_basic_info().converted_type() {
- // List types
- ConvertedType::LIST => {
- assert_eq!(
- field.get_fields().len(),
- 1,
- "Invalid list type {:?}",
- field
- );
-
- let repeated_field = field.get_fields()[0].clone();
- assert_eq!(
- repeated_field.get_basic_info().repetition(),
- Repetition::REPEATED,
- "Invalid list type {:?}",
- field
- );
-
- if Reader::is_element_type(&repeated_field) {
- // Support for backward compatible lists
- let reader = self.reader_tree(
- repeated_field,
- &mut path,
- curr_def_level,
- curr_rep_level,
- paths,
- row_group_reader,
- );
-
- Reader::RepeatedReader(
- field,
- curr_def_level,
- curr_rep_level,
- Box::new(reader),
- )
- } else {
- let child_field = repeated_field.get_fields()[0].clone();
-
- path.push(String::from(repeated_field.name()));
-
- let reader = self.reader_tree(
- child_field,
- &mut path,
- curr_def_level + 1,
- curr_rep_level + 1,
- paths,
- row_group_reader,
- );
-
- path.pop();
-
- Reader::RepeatedReader(
- field,
- curr_def_level,
- curr_rep_level,
- Box::new(reader),
- )
- }
- }
- // Map types (key-value pairs)
- ConvertedType::MAP | ConvertedType::MAP_KEY_VALUE => {
- assert_eq!(
- field.get_fields().len(),
- 1,
- "Invalid map type: {:?}",
- field
- );
- assert!(
- !field.get_fields()[0].is_primitive(),
- "Invalid map type: {:?}",
- field
- );
-
- let key_value_type = field.get_fields()[0].clone();
- assert_eq!(
- key_value_type.get_basic_info().repetition(),
- Repetition::REPEATED,
- "Invalid map type: {:?}",
- field
- );
- assert_eq!(
- key_value_type.get_fields().len(),
- 2,
- "Invalid map type: {:?}",
- field
- );
-
- path.push(String::from(key_value_type.name()));
-
- let key_type = &key_value_type.get_fields()[0];
- assert!(
- key_type.is_primitive(),
- "Map key type is expected to be a primitive type, but found {:?}",
- key_type
- );
- let key_reader = self.reader_tree(
- key_type.clone(),
- &mut path,
- curr_def_level + 1,
- curr_rep_level + 1,
- paths,
- row_group_reader,
- );
-
- let value_type = &key_value_type.get_fields()[1];
- let value_reader = self.reader_tree(
- value_type.clone(),
- &mut path,
- curr_def_level + 1,
- curr_rep_level + 1,
- paths,
- row_group_reader,
- );
-
- path.pop();
-
- Reader::KeyValueReader(
- field,
- curr_def_level,
- curr_rep_level,
- Box::new(key_reader),
- Box::new(value_reader),
- )
- }
- // A repeated field that is neither contained by a `LIST`- or
- // `MAP`-annotated group nor annotated by `LIST` or `MAP`
- // should be interpreted as a required list of required
- // elements where the element type is the type of the field.
- _ if repetition == Repetition::REPEATED => {
- let required_field = Type::group_type_builder(field.name())
- .with_repetition(Repetition::REQUIRED)
- .with_converted_type(field.get_basic_info().converted_type())
- .with_fields(&mut Vec::from(field.get_fields()))
- .build()
- .unwrap();
-
- path.pop();
-
- let reader = self.reader_tree(
- Arc::new(required_field),
- &mut path,
- curr_def_level,
- curr_rep_level,
- paths,
- row_group_reader,
- );
-
- Reader::RepeatedReader(
- field,
- curr_def_level - 1,
- curr_rep_level - 1,
- Box::new(reader),
- )
- }
- // Group types (structs)
- _ => {
- let mut readers = Vec::new();
- for child in field.get_fields() {
- let reader = self.reader_tree(
- child.clone(),
- &mut path,
- curr_def_level,
- curr_rep_level,
- paths,
- row_group_reader,
- );
- readers.push(reader);
- }
- Reader::GroupReader(Some(field), curr_def_level, readers)
- }
- }
- };
- path.pop();
-
- Reader::option(repetition, curr_def_level, reader)
- }
-}
-
-/// Reader tree for record assembly
-pub enum Reader {
- // Primitive reader with type information and triplet iterator
- PrimitiveReader(TypePtr, TripletIter),
- // Optional reader with definition level of a parent and a reader
- OptionReader(i16, Box<Reader>),
- // Group (struct) reader with type information, definition level and list of child
- // readers. When it represents message type, type information is None
- GroupReader(Option<TypePtr>, i16, Vec<Reader>),
- // Reader for repeated values, e.g. lists, contains type information, definition
- // level, repetition level and a child reader
- RepeatedReader(TypePtr, i16, i16, Box<Reader>),
- // Reader of key-value pairs, e.g. maps, contains type information, definition
- // level, repetition level, child reader for keys and child reader for values
- KeyValueReader(TypePtr, i16, i16, Box<Reader>, Box<Reader>),
-}
-
-impl Reader {
- /// Wraps reader in option reader based on repetition.
- fn option(repetition: Repetition, def_level: i16, reader: Reader) -> Self {
- if repetition == Repetition::OPTIONAL {
- Reader::OptionReader(def_level - 1, Box::new(reader))
- } else {
- reader
- }
- }
-
- /// Returns true if repeated type is an element type for the list.
- /// Used to determine legacy list types.
- /// This method is copied from Spark Parquet reader and is based on the reference:
- /// <https://github.com/apache/parquet-format/blob/master/LogicalTypes.md>
- /// #backward-compatibility-rules
- fn is_element_type(repeated_type: &Type) -> bool {
- // For legacy 2-level list types with primitive element type, e.g.:
- //
- // // ARRAY<INT> (nullable list, non-null elements)
- // optional group my_list (LIST) {
- // repeated int32 element;
- // }
- //
- repeated_type.is_primitive() ||
- // For legacy 2-level list types whose element type is a group type with 2 or more
- // fields, e.g.:
- //
- // // ARRAY<STRUCT<str: STRING, num: INT>> (nullable list, non-null elements)
- // optional group my_list (LIST) {
- // repeated group element {
- // required binary str (UTF8);
- // required int32 num;
- // };
- // }
- //
- repeated_type.is_group() && repeated_type.get_fields().len() > 1 ||
- // For legacy 2-level list types generated by parquet-avro (Parquet version < 1.6.0),
- // e.g.:
- //
- // // ARRAY<STRUCT<str: STRING>> (nullable list, non-null elements)
- // optional group my_list (LIST) {
- // repeated group array {
- // required binary str (UTF8);
- // };
- // }
- //
- repeated_type.name() == "array" ||
- // For Parquet data generated by parquet-thrift, e.g.:
- //
- // // ARRAY<STRUCT<str: STRING>> (nullable list, non-null elements)
- // optional group my_list (LIST) {
- // repeated group my_list_tuple {
- // required binary str (UTF8);
- // };
- // }
- //
- repeated_type.name().ends_with("_tuple")
- }
-
- /// Reads current record as `Row` from the reader tree.
- /// Automatically advances all necessary readers.
- /// This must be called on the root level reader (i.e., for Message type).
- /// Otherwise, it will panic.
- fn read(&mut self) -> Row {
- match *self {
- Reader::GroupReader(_, _, ref mut readers) => {
- let mut fields = Vec::new();
- for reader in readers {
- fields.push((String::from(reader.field_name()), reader.read_field()));
- }
- make_row(fields)
- }
- _ => panic!("Cannot call read() on {}", self),
- }
- }
-
- /// Reads current record as `Field` from the reader tree.
- /// Automatically advances all necessary readers.
- fn read_field(&mut self) -> Field {
- match *self {
- Reader::PrimitiveReader(_, ref mut column) => {
- let value = column.current_value();
- column.read_next().unwrap();
- value
- }
- Reader::OptionReader(def_level, ref mut reader) => {
- if reader.current_def_level() > def_level {
- reader.read_field()
- } else {
- reader.advance_columns();
- Field::Null
- }
- }
- Reader::GroupReader(_, def_level, ref mut readers) => {
- let mut fields = Vec::new();
- for reader in readers {
- if reader.repetition() != Repetition::OPTIONAL
- || reader.current_def_level() > def_level
- {
- fields.push((
- String::from(reader.field_name()),
- reader.read_field(),
- ));
- } else {
- reader.advance_columns();
- fields.push((String::from(reader.field_name()), Field::Null));
- }
- }
- let row = make_row(fields);
- Field::Group(row)
- }
- Reader::RepeatedReader(_, def_level, rep_level, ref mut reader) => {
- let mut elements = Vec::new();
- loop {
- if reader.current_def_level() > def_level {
- elements.push(reader.read_field());
- } else {
- reader.advance_columns();
- // If the current definition level is equal to the definition
- // level of this repeated type, then the
- // result is an empty list and the repetition level
- // will always be <= rl.
- break;
- }
-
- // This covers case when we are out of repetition levels and should
- // close the group, or there are no values left to
- // buffer.
- if !reader.has_next() || reader.current_rep_level() <= rep_level {
- break;
- }
- }
- Field::ListInternal(make_list(elements))
- }
- Reader::KeyValueReader(
- _,
- def_level,
- rep_level,
- ref mut keys,
- ref mut values,
- ) => {
- let mut pairs = Vec::new();
- loop {
- if keys.current_def_level() > def_level {
- pairs.push((keys.read_field(), values.read_field()));
- } else {
- keys.advance_columns();
- values.advance_columns();
- // If the current definition level is equal to the definition
- // level of this repeated type, then the
- // result is an empty list and the repetition level
- // will always be <= rl.
- break;
- }
-
- // This covers case when we are out of repetition levels and should
- // close the group, or there are no values left to
- // buffer.
- if !keys.has_next() || keys.current_rep_level() <= rep_level {
- break;
- }
- }
-
- Field::MapInternal(make_map(pairs))
- }
- }
- }
-
- /// Returns field name for the current reader.
- fn field_name(&self) -> &str {
- match *self {
- Reader::PrimitiveReader(ref field, _) => field.name(),
- Reader::OptionReader(_, ref reader) => reader.field_name(),
- Reader::GroupReader(ref opt, ..) => match opt {
- Some(ref field) => field.name(),
- None => panic!("Field is None for group reader"),
- },
- Reader::RepeatedReader(ref field, ..) => field.name(),
- Reader::KeyValueReader(ref field, ..) => field.name(),
- }
- }
-
- /// Returns repetition for the current reader.
- fn repetition(&self) -> Repetition {
- match *self {
- Reader::PrimitiveReader(ref field, _) => field.get_basic_info().repetition(),
- Reader::OptionReader(_, ref reader) => reader.repetition(),
- Reader::GroupReader(ref opt, ..) => match opt {
- Some(ref field) => field.get_basic_info().repetition(),
- None => panic!("Field is None for group reader"),
- },
- Reader::RepeatedReader(ref field, ..) => field.get_basic_info().repetition(),
- Reader::KeyValueReader(ref field, ..) => field.get_basic_info().repetition(),
- }
- }
-
- /// Returns true, if current reader has more values, false otherwise.
- /// Method does not advance internal iterator.
- fn has_next(&self) -> bool {
- match *self {
- Reader::PrimitiveReader(_, ref column) => column.has_next(),
- Reader::OptionReader(_, ref reader) => reader.has_next(),
- Reader::GroupReader(_, _, ref readers) => readers.first().unwrap().has_next(),
- Reader::RepeatedReader(_, _, _, ref reader) => reader.has_next(),
- Reader::KeyValueReader(_, _, _, ref keys, _) => keys.has_next(),
- }
- }
-
- /// Returns current definition level,
- /// Method does not advance internal iterator.
- fn current_def_level(&self) -> i16 {
- match *self {
- Reader::PrimitiveReader(_, ref column) => column.current_def_level(),
- Reader::OptionReader(_, ref reader) => reader.current_def_level(),
- Reader::GroupReader(_, _, ref readers) => match readers.first() {
- Some(reader) => reader.current_def_level(),
- None => panic!("Current definition level: empty group reader"),
- },
- Reader::RepeatedReader(_, _, _, ref reader) => reader.current_def_level(),
- Reader::KeyValueReader(_, _, _, ref keys, _) => keys.current_def_level(),
- }
- }
-
- /// Returns current repetition level.
- /// Method does not advance internal iterator.
- fn current_rep_level(&self) -> i16 {
- match *self {
- Reader::PrimitiveReader(_, ref column) => column.current_rep_level(),
- Reader::OptionReader(_, ref reader) => reader.current_rep_level(),
- Reader::GroupReader(_, _, ref readers) => match readers.first() {
- Some(reader) => reader.current_rep_level(),
- None => panic!("Current repetition level: empty group reader"),
- },
- Reader::RepeatedReader(_, _, _, ref reader) => reader.current_rep_level(),
- Reader::KeyValueReader(_, _, _, ref keys, _) => keys.current_rep_level(),
- }
- }
-
- /// Advances leaf columns for the current reader.
- fn advance_columns(&mut self) {
- match *self {
- Reader::PrimitiveReader(_, ref mut column) => {
- column.read_next().unwrap();
- }
- Reader::OptionReader(_, ref mut reader) => {
- reader.advance_columns();
- }
- Reader::GroupReader(_, _, ref mut readers) => {
- for reader in readers {
- reader.advance_columns();
- }
- }
- Reader::RepeatedReader(_, _, _, ref mut reader) => {
- reader.advance_columns();
- }
- Reader::KeyValueReader(_, _, _, ref mut keys, ref mut values) => {
- keys.advance_columns();
- values.advance_columns();
- }
- }
- }
-}
-
-impl fmt::Display for Reader {
- fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
- let s = match self {
- Reader::PrimitiveReader(..) => "PrimitiveReader",
- Reader::OptionReader(..) => "OptionReader",
- Reader::GroupReader(..) => "GroupReader",
- Reader::RepeatedReader(..) => "RepeatedReader",
- Reader::KeyValueReader(..) => "KeyValueReader",
- };
- write!(f, "{}", s)
- }
-}
-
-// ----------------------------------------------------------------------
-// Row iterators
-
-/// The enum Either with variants That represet a reference and a box of
-/// [`FileReader`](crate::file::reader::FileReader).
-enum Either<'a> {
- Left(&'a dyn FileReader),
- Right(Box<dyn FileReader>),
-}
-
-impl<'a> Either<'a> {
- fn reader(&self) -> &dyn FileReader {
- match *self {
- Either::Left(r) => r,
- Either::Right(ref r) => &**r,
- }
- }
-}
-
-/// Iterator of [`Row`](crate::record::Row)s.
-/// It is used either for a single row group to iterate over data in that row group, or
-/// an entire file with auto buffering of all row groups.
-pub struct RowIter<'a> {
- descr: SchemaDescPtr,
- tree_builder: TreeBuilder,
- file_reader: Option<Either<'a>>,
- current_row_group: usize,
- num_row_groups: usize,
- row_iter: Option<ReaderIter>,
-}
-
-impl<'a> RowIter<'a> {
- /// Creates a new iterator of [`Row`](crate::record::Row)s.
- fn new(
- file_reader: Option<Either<'a>>,
- row_iter: Option<ReaderIter>,
- descr: SchemaDescPtr,
- ) -> Self {
- let tree_builder = Self::tree_builder();
- let num_row_groups = match file_reader {
- Some(ref r) => r.reader().num_row_groups(),
- None => 0,
- };
-
- Self {
- descr,
- file_reader,
- tree_builder,
- num_row_groups,
- row_iter,
- current_row_group: 0,
- }
- }
-
- /// Creates iterator of [`Row`](crate::record::Row)s for all row groups in a
- /// file.
- pub fn from_file(proj: Option<Type>, reader: &'a dyn FileReader) -> Result<Self> {
- let either = Either::Left(reader);
- let descr = Self::get_proj_descr(
- proj,
- reader.metadata().file_metadata().schema_descr_ptr(),
- )?;
-
- Ok(Self::new(Some(either), None, descr))
- }
-
- /// Creates iterator of [`Row`](crate::record::Row)s for a specific row group.
- pub fn from_row_group(
- proj: Option<Type>,
- reader: &'a dyn RowGroupReader,
- ) -> Result<Self> {
- let descr = Self::get_proj_descr(proj, reader.metadata().schema_descr_ptr())?;
- let tree_builder = Self::tree_builder();
- let row_iter = tree_builder.as_iter(descr.clone(), reader);
-
- // For row group we need to set `current_row_group` >= `num_row_groups`, because
- // we only have one row group and can't buffer more.
- Ok(Self::new(None, Some(row_iter), descr))
- }
-
- /// Creates a iterator of [`Row`](crate::record::Row)s from a
- /// [`FileReader`](crate::file::reader::FileReader) using the full file schema.
- pub fn from_file_into(reader: Box<dyn FileReader>) -> Self {
- let either = Either::Right(reader);
- let descr = either
- .reader()
- .metadata()
- .file_metadata()
- .schema_descr_ptr();
-
- Self::new(Some(either), None, descr)
- }
-
- /// Tries to create a iterator of [`Row`](crate::record::Row)s using projections.
- /// Returns a error if a file reader is not the source of this iterator.
- ///
- /// The Projected schema can be a subset of or equal to the file schema,
- /// when it is None, full file schema is assumed.
- pub fn project(self, proj: Option<Type>) -> Result<Self> {
- match self.file_reader {
- Some(ref either) => {
- let schema = either
- .reader()
- .metadata()
- .file_metadata()
- .schema_descr_ptr();
- let descr = Self::get_proj_descr(proj, schema)?;
-
- Ok(Self::new(self.file_reader, None, descr))
- }
- None => Err(general_err!("File reader is required to use projections")),
- }
- }
-
- /// Helper method to get schema descriptor for projected schema.
- /// If projection is None, then full schema is returned.
- #[inline]
- fn get_proj_descr(
- proj: Option<Type>,
- root_descr: SchemaDescPtr,
- ) -> Result<SchemaDescPtr> {
- match proj {
- Some(projection) => {
- // check if projection is part of file schema
- let root_schema = root_descr.root_schema();
- if !root_schema.check_contains(&projection) {
- return Err(general_err!("Root schema does not contain projection"));
- }
- Ok(Arc::new(SchemaDescriptor::new(Arc::new(projection))))
- }
- None => Ok(root_descr),
- }
- }
-
- /// Returns common tree builder, so the same settings are applied to both iterators
- /// from file reader and row group.
- #[inline]
- fn tree_builder() -> TreeBuilder {
- TreeBuilder::new()
- }
-}
-
-impl<'a> Iterator for RowIter<'a> {
- type Item = Row;
-
- fn next(&mut self) -> Option<Row> {
- let mut row = None;
- if let Some(ref mut iter) = self.row_iter {
- row = iter.next();
- }
-
- while row.is_none() && self.current_row_group < self.num_row_groups {
- // We do not expect any failures when accessing a row group, and file reader
- // must be set for selecting next row group.
- if let Some(ref either) = self.file_reader {
- let file_reader = either.reader();
- let row_group_reader = &*file_reader
- .get_row_group(self.current_row_group)
- .expect("Row group is required to advance");
-
- let mut iter = self
- .tree_builder
- .as_iter(self.descr.clone(), row_group_reader);
-
- row = iter.next();
-
- self.current_row_group += 1;
- self.row_iter = Some(iter);
- }
- }
-
- row
- }
-}
-
-/// Internal iterator of [`Row`](crate::record::Row)s for a reader.
-pub struct ReaderIter {
- root_reader: Reader,
- records_left: usize,
-}
-
-impl ReaderIter {
- fn new(mut root_reader: Reader, num_records: usize) -> Self {
- // Prepare root reader by advancing all column vectors
- root_reader.advance_columns();
- Self {
- root_reader,
- records_left: num_records,
- }
- }
-}
-
-impl Iterator for ReaderIter {
- type Item = Row;
-
- fn next(&mut self) -> Option<Row> {
- if self.records_left > 0 {
- self.records_left -= 1;
- Some(self.root_reader.read())
- } else {
- None
- }
- }
-}
-
-#[cfg(test)]
-mod tests {
- use super::*;
-
- use crate::errors::{ParquetError, Result};
- use crate::file::reader::{FileReader, SerializedFileReader};
- use crate::record::api::{Field, Row, RowAccessor, RowFormatter};
- use crate::schema::parser::parse_message_type;
- use crate::util::test_common::{get_test_file, get_test_path};
- use std::convert::TryFrom;
-
- // Convenient macros to assemble row, list, map, and group.
-
- macro_rules! row {
- () => {
- {
- let result = Vec::new();
- make_row(result)
- }
- };
- ( $( $e:expr ), + ) => {
- {
- let mut result = Vec::new();
- $(
- result.push($e);
- )*
- make_row(result)
- }
- }
- }
-
- macro_rules! list {
- () => {
- {
- let result = Vec::new();
- Field::ListInternal(make_list(result))
- }
- };
- ( $( $e:expr ), + ) => {
- {
- let mut result = Vec::new();
- $(
- result.push($e);
- )*
- Field::ListInternal(make_list(result))
- }
- }
- }
-
- macro_rules! map {
- () => {
- {
- let result = Vec::new();
- Field::MapInternal(make_map(result))
- }
- };
- ( $( $e:expr ), + ) => {
- {
- let mut result = Vec::new();
- $(
- result.push($e);
- )*
- Field::MapInternal(make_map(result))
- }
- }
- }
-
- macro_rules! group {
- ( $( $e:expr ), * ) => {
- {
- Field::Group(row!($( $e ), *))
- }
- }
- }
-
- #[test]
- fn test_file_reader_rows_nulls() {
- let rows = test_file_reader_rows("nulls.snappy.parquet", None).unwrap();
- let expected_rows = vec![
- row![(
- "b_struct".to_string(),
- group![("b_c_int".to_string(), Field::Null)]
- )],
- row![(
- "b_struct".to_string(),
- group![("b_c_int".to_string(), Field::Null)]
- )],
- row![(
- "b_struct".to_string(),
- group![("b_c_int".to_string(), Field::Null)]
- )],
- row![(
- "b_struct".to_string(),
- group![("b_c_int".to_string(), Field::Null)]
- )],
- row![(
- "b_struct".to_string(),
- group![("b_c_int".to_string(), Field::Null)]
- )],
- row![(
- "b_struct".to_string(),
- group![("b_c_int".to_string(), Field::Null)]
- )],
- row![(
- "b_struct".to_string(),
- group![("b_c_int".to_string(), Field::Null)]
- )],
- row![(
- "b_struct".to_string(),
- group![("b_c_int".to_string(), Field::Null)]
- )],
- ];
- assert_eq!(rows, expected_rows);
- }
-
- #[test]
- fn test_file_reader_rows_nonnullable() {
- let rows = test_file_reader_rows("nonnullable.impala.parquet", None).unwrap();
- let expected_rows = vec![row![
- ("ID".to_string(), Field::Long(8)),
- ("Int_Array".to_string(), list![Field::Int(-1)]),
- (
- "int_array_array".to_string(),
- list![list![Field::Int(-1), Field::Int(-2)], list![]]
- ),
- (
- "Int_Map".to_string(),
- map![(Field::Str("k1".to_string()), Field::Int(-1))]
- ),
- (
- "int_map_array".to_string(),
- list![
- map![],
- map![(Field::Str("k1".to_string()), Field::Int(1))],
- map![],
- map![]
- ]
- ),
- (
- "nested_Struct".to_string(),
- group![
- ("a".to_string(), Field::Int(-1)),
- ("B".to_string(), list![Field::Int(-1)]),
- (
- "c".to_string(),
- group![(
- "D".to_string(),
- list![list![group![
- ("e".to_string(), Field::Int(-1)),
- ("f".to_string(), Field::Str("nonnullable".to_string()))
- ]]]
- )]
- ),
- ("G".to_string(), map![])
- ]
- )
- ]];
- assert_eq!(rows, expected_rows);
- }
-
- #[test]
- fn test_file_reader_rows_nullable() {
- let rows = test_file_reader_rows("nullable.impala.parquet", None).unwrap();
- let expected_rows = vec![
- row![
- ("id".to_string(), Field::Long(1)),
- (
- "int_array".to_string(),
- list![Field::Int(1), Field::Int(2), Field::Int(3)]
- ),
- (
- "int_array_Array".to_string(),
- list![
- list![Field::Int(1), Field::Int(2)],
- list![Field::Int(3), Field::Int(4)]
- ]
- ),
- (
- "int_map".to_string(),
- map![
- (Field::Str("k1".to_string()), Field::Int(1)),
- (Field::Str("k2".to_string()), Field::Int(100))
- ]
- ),
- (
- "int_Map_Array".to_string(),
- list![map![(Field::Str("k1".to_string()), Field::Int(1))]]
- ),
- (
- "nested_struct".to_string(),
- group![
- ("A".to_string(), Field::Int(1)),
- ("b".to_string(), list![Field::Int(1)]),
- (
- "C".to_string(),
- group![(
- "d".to_string(),
- list![
- list![
- group![
- ("E".to_string(), Field::Int(10)),
- (
- "F".to_string(),
- Field::Str("aaa".to_string())
- )
- ],
- group![
- ("E".to_string(), Field::Int(-10)),
- (
- "F".to_string(),
- Field::Str("bbb".to_string())
- )
- ]
- ],
- list![group![
- ("E".to_string(), Field::Int(11)),
- ("F".to_string(), Field::Str("c".to_string()))
- ]]
- ]
- )]
- ),
- (
- "g".to_string(),
- map![(
- Field::Str("foo".to_string()),
- group![(
- "H".to_string(),
- group![("i".to_string(), list![Field::Double(1.1)])]
- )]
- )]
- )
- ]
- )
- ],
- row![
- ("id".to_string(), Field::Long(2)),
- (
- "int_array".to_string(),
- list![
- Field::Null,
- Field::Int(1),
- Field::Int(2),
- Field::Null,
- Field::Int(3),
- Field::Null
- ]
- ),
- (
- "int_array_Array".to_string(),
- list![
- list![Field::Null, Field::Int(1), Field::Int(2), Field::Null],
- list![Field::Int(3), Field::Null, Field::Int(4)],
- list![],
- Field::Null
- ]
- ),
- (
- "int_map".to_string(),
- map![
- (Field::Str("k1".to_string()), Field::Int(2)),
- (Field::Str("k2".to_string()), Field::Null)
- ]
- ),
- (
- "int_Map_Array".to_string(),
- list![
- map![
- (Field::Str("k3".to_string()), Field::Null),
- (Field::Str("k1".to_string()), Field::Int(1))
- ],
- Field::Null,
- map![]
- ]
- ),
- (
- "nested_struct".to_string(),
- group![
- ("A".to_string(), Field::Null),
- ("b".to_string(), list![Field::Null]),
- (
- "C".to_string(),
- group![(
- "d".to_string(),
- list![
- list![
- group![
- ("E".to_string(), Field::Null),
- ("F".to_string(), Field::Null)
- ],
- group![
- ("E".to_string(), Field::Int(10)),
- (
- "F".to_string(),
- Field::Str("aaa".to_string())
- )
- ],
- group![
- ("E".to_string(), Field::Null),
- ("F".to_string(), Field::Null)
- ],
- group![
- ("E".to_string(), Field::Int(-10)),
- (
- "F".to_string(),
- Field::Str("bbb".to_string())
- )
- ],
- group![
- ("E".to_string(), Field::Null),
- ("F".to_string(), Field::Null)
- ]
- ],
- list![
- group![
- ("E".to_string(), Field::Int(11)),
- (
- "F".to_string(),
- Field::Str("c".to_string())
- )
- ],
- Field::Null
- ],
- list![],
- Field::Null
- ]
- )]
- ),
- (
- "g".to_string(),
- map![
- (
- Field::Str("g1".to_string()),
- group![(
- "H".to_string(),
- group![(
- "i".to_string(),
- list![Field::Double(2.2), Field::Null]
- )]
- )]
- ),
- (
- Field::Str("g2".to_string()),
- group![(
- "H".to_string(),
- group![("i".to_string(), list![])]
- )]
- ),
- (Field::Str("g3".to_string()), Field::Null),
- (
- Field::Str("g4".to_string()),
- group![(
- "H".to_string(),
- group![("i".to_string(), Field::Null)]
- )]
- ),
- (
- Field::Str("g5".to_string()),
- group![("H".to_string(), Field::Null)]
- )
- ]
- )
- ]
- )
- ],
- row![
- ("id".to_string(), Field::Long(3)),
- ("int_array".to_string(), list![]),
- ("int_array_Array".to_string(), list![Field::Null]),
- ("int_map".to_string(), map![]),
- ("int_Map_Array".to_string(), list![Field::Null, Field::Null]),
- (
- "nested_struct".to_string(),
- group![
- ("A".to_string(), Field::Null),
- ("b".to_string(), Field::Null),
- ("C".to_string(), group![("d".to_string(), list![])]),
- ("g".to_string(), map![])
- ]
- )
- ],
- row![
- ("id".to_string(), Field::Long(4)),
- ("int_array".to_string(), Field::Null),
- ("int_array_Array".to_string(), list![]),
- ("int_map".to_string(), map![]),
- ("int_Map_Array".to_string(), list![]),
- (
- "nested_struct".to_string(),
- group![
- ("A".to_string(), Field::Null),
- ("b".to_string(), Field::Null),
- ("C".to_string(), group![("d".to_string(), Field::Null)]),
- ("g".to_string(), Field::Null)
- ]
- )
- ],
- row![
- ("id".to_string(), Field::Long(5)),
- ("int_array".to_string(), Field::Null),
- ("int_array_Array".to_string(), Field::Null),
- ("int_map".to_string(), map![]),
- ("int_Map_Array".to_string(), Field::Null),
- (
- "nested_struct".to_string(),
- group![
- ("A".to_string(), Field::Null),
- ("b".to_string(), Field::Null),
- ("C".to_string(), Field::Null),
- (
- "g".to_string(),
- map![(
- Field::Str("foo".to_string()),
- group![(
- "H".to_string(),
- group![(
- "i".to_string(),
- list![Field::Double(2.2), Field::Double(3.3)]
- )]
- )]
- )]
- )
- ]
- )
- ],
- row![
- ("id".to_string(), Field::Long(6)),
- ("int_array".to_string(), Field::Null),
- ("int_array_Array".to_string(), Field::Null),
- ("int_map".to_string(), Field::Null),
- ("int_Map_Array".to_string(), Field::Null),
- ("nested_struct".to_string(), Field::Null)
- ],
- row![
- ("id".to_string(), Field::Long(7)),
- ("int_array".to_string(), Field::Null),
- (
- "int_array_Array".to_string(),
- list![Field::Null, list![Field::Int(5), Field::Int(6)]]
- ),
- (
- "int_map".to_string(),
- map![
- (Field::Str("k1".to_string()), Field::Null),
- (Field::Str("k3".to_string()), Field::Null)
- ]
- ),
- ("int_Map_Array".to_string(), Field::Null),
- (
- "nested_struct".to_string(),
- group![
- ("A".to_string(), Field::Int(7)),
- (
- "b".to_string(),
- list![Field::Int(2), Field::Int(3), Field::Null]
- ),
- (
- "C".to_string(),
- group![(
- "d".to_string(),
- list![list![], list![Field::Null], Field::Null]
- )]
- ),
- ("g".to_string(), Field::Null)
- ]
- )
- ],
- ];
- assert_eq!(rows, expected_rows);
- }
-
- #[test]
- fn test_file_reader_rows_projection() {
- let schema = "
- message spark_schema {
- REQUIRED DOUBLE c;
- REQUIRED INT32 b;
- }
- ";
- let schema = parse_message_type(&schema).unwrap();
- let rows =
- test_file_reader_rows("nested_maps.snappy.parquet", Some(schema)).unwrap();
- let expected_rows = vec![
- row![
- ("c".to_string(), Field::Double(1.0)),
- ("b".to_string(), Field::Int(1))
- ],
- row![
- ("c".to_string(), Field::Double(1.0)),
- ("b".to_string(), Field::Int(1))
- ],
- row![
- ("c".to_string(), Field::Double(1.0)),
- ("b".to_string(), Field::Int(1))
- ],
- row![
- ("c".to_string(), Field::Double(1.0)),
- ("b".to_string(), Field::Int(1))
- ],
- row![
- ("c".to_string(), Field::Double(1.0)),
- ("b".to_string(), Field::Int(1))
- ],
- row![
- ("c".to_string(), Field::Double(1.0)),
- ("b".to_string(), Field::Int(1))
- ],
- ];
- assert_eq!(rows, expected_rows);
- }
-
- #[test]
- fn test_iter_columns_in_row() {
- let r = row![
- ("c".to_string(), Field::Double(1.0)),
- ("b".to_string(), Field::Int(1))
- ];
- let mut result = Vec::new();
- for (name, record) in r.get_column_iter() {
- result.push((name, record));
- }
- assert_eq!(
- vec![
- (&"c".to_string(), &Field::Double(1.0)),
- (&"b".to_string(), &Field::Int(1))
- ],
- result
- );
- }
-
- #[test]
- fn test_file_reader_rows_projection_map() {
- let schema = "
- message spark_schema {
- OPTIONAL group a (MAP) {
- REPEATED group key_value {
- REQUIRED BYTE_ARRAY key (UTF8);
- OPTIONAL group value (MAP) {
- REPEATED group key_value {
- REQUIRED INT32 key;
- REQUIRED BOOLEAN value;
- }
- }
- }
- }
- }
- ";
- let schema = parse_message_type(&schema).unwrap();
- let rows =
- test_file_reader_rows("nested_maps.snappy.parquet", Some(schema)).unwrap();
- let expected_rows = vec![
- row![(
- "a".to_string(),
- map![(
- Field::Str("a".to_string()),
- map![
- (Field::Int(1), Field::Bool(true)),
- (Field::Int(2), Field::Bool(false))
- ]
- )]
- )],
- row![(
- "a".to_string(),
- map![(
- Field::Str("b".to_string()),
- map![(Field::Int(1), Field::Bool(true))]
- )]
- )],
- row![(
- "a".to_string(),
- map![(Field::Str("c".to_string()), Field::Null)]
- )],
- row![("a".to_string(), map![(Field::Str("d".to_string()), map![])])],
- row![(
- "a".to_string(),
- map![(
- Field::Str("e".to_string()),
- map![(Field::Int(1), Field::Bool(true))]
- )]
- )],
- row![(
- "a".to_string(),
- map![(
- Field::Str("f".to_string()),
- map![
- (Field::Int(3), Field::Bool(true)),
- (Field::Int(4), Field::Bool(false)),
- (Field::Int(5), Field::Bool(true))
- ]
- )]
- )],
- ];
- assert_eq!(rows, expected_rows);
- }
-
- #[test]
- fn test_file_reader_rows_projection_list() {
- let schema = "
- message spark_schema {
- OPTIONAL group a (LIST) {
- REPEATED group list {
- OPTIONAL group element (LIST) {
- REPEATED group list {
- OPTIONAL group element (LIST) {
- REPEATED group list {
- OPTIONAL BYTE_ARRAY element (UTF8);
- }
- }
- }
- }
- }
- }
- }
- ";
- let schema = parse_message_type(&schema).unwrap();
- let rows =
- test_file_reader_rows("nested_lists.snappy.parquet", Some(schema)).unwrap();
- let expected_rows = vec![
- row![(
- "a".to_string(),
- list![
- list![
- list![Field::Str("a".to_string()), Field::Str("b".to_string())],
- list![Field::Str("c".to_string())]
- ],
- list![Field::Null, list![Field::Str("d".to_string())]]
- ]
- )],
- row![(
- "a".to_string(),
- list![
- list![
- list![Field::Str("a".to_string()), Field::Str("b".to_string())],
- list![Field::Str("c".to_string()), Field::Str("d".to_string())]
- ],
- list![Field::Null, list![Field::Str("e".to_string())]]
- ]
- )],
- row![(
- "a".to_string(),
- list![
- list![
- list![Field::Str("a".to_string()), Field::Str("b".to_string())],
- list![Field::Str("c".to_string()), Field::Str("d".to_string())],
- list![Field::Str("e".to_string())]
- ],
- list![Field::Null, list![Field::Str("f".to_string())]]
- ]
- )],
- ];
- assert_eq!(rows, expected_rows);
- }
-
- #[test]
- fn test_file_reader_rows_invalid_projection() {
- let schema = "
- message spark_schema {
- REQUIRED INT32 key;
- REQUIRED BOOLEAN value;
- }
- ";
- let schema = parse_message_type(&schema).unwrap();
- let res = test_file_reader_rows("nested_maps.snappy.parquet", Some(schema));
- assert!(res.is_err());
- assert_eq!(
- res.unwrap_err(),
- general_err!("Root schema does not contain projection")
- );
- }
-
- #[test]
- fn test_row_group_rows_invalid_projection() {
- let schema = "
- message spark_schema {
- REQUIRED INT32 key;
- REQUIRED BOOLEAN value;
- }
- ";
- let schema = parse_message_type(&schema).unwrap();
- let res = test_row_group_rows("nested_maps.snappy.parquet", Some(schema));
- assert!(res.is_err());
- assert_eq!(
- res.unwrap_err(),
- general_err!("Root schema does not contain projection")
- );
- }
-
- #[test]
- #[should_panic(expected = "Invalid map type")]
- fn test_file_reader_rows_invalid_map_type() {
- let schema = "
- message spark_schema {
- OPTIONAL group a (MAP) {
- REPEATED group key_value {
- REQUIRED BYTE_ARRAY key (UTF8);
- OPTIONAL group value (MAP) {
- REPEATED group key_value {
- REQUIRED INT32 key;
- }
- }
- }
- }
- }
- ";
- let schema = parse_message_type(&schema).unwrap();
- test_file_reader_rows("nested_maps.snappy.parquet", Some(schema)).unwrap();
- }
-
- #[test]
- fn test_file_reader_iter() {
- let path = get_test_path("alltypes_plain.parquet");
- let vec = vec![path]
- .iter()
- .map(|p| SerializedFileReader::try_from(p.as_path()).unwrap())
- .flat_map(|r| RowIter::from_file_into(Box::new(r)))
- .flat_map(|r| r.get_int(0))
- .collect::<Vec<_>>();
-
- assert_eq!(vec, vec![4, 5, 6, 7, 2, 3, 0, 1]);
- }
-
- #[test]
- fn test_file_reader_iter_projection() {
- let path = get_test_path("alltypes_plain.parquet");
- let values = vec![path]
- .iter()
- .map(|p| SerializedFileReader::try_from(p.as_path()).unwrap())
- .flat_map(|r| {
- let schema = "message schema { OPTIONAL INT32 id; }";
- let proj = parse_message_type(&schema).ok();
-
- RowIter::from_file_into(Box::new(r)).project(proj).unwrap()
- })
- .map(|r| format!("id:{}", r.fmt(0)))
- .collect::<Vec<_>>()
- .join(", ");
-
- assert_eq!(values, "id:4, id:5, id:6, id:7, id:2, id:3, id:0, id:1");
- }
-
- #[test]
- fn test_file_reader_iter_projection_err() {
- let schema = "
- message spark_schema {
- REQUIRED INT32 key;
- REQUIRED BOOLEAN value;
- }
- ";
- let proj = parse_message_type(&schema).ok();
- let path = get_test_path("nested_maps.snappy.parquet");
- let reader = SerializedFileReader::try_from(path.as_path()).unwrap();
- let res = RowIter::from_file_into(Box::new(reader)).project(proj);
-
- assert!(res.is_err());
- assert_eq!(
- res.err().unwrap(),
- general_err!("Root schema does not contain projection")
- );
- }
-
- #[test]
- fn test_tree_reader_handle_repeated_fields_with_no_annotation() {
- // Array field `phoneNumbers` does not contain LIST annotation.
- // We parse it as struct with `phone` repeated field as array.
- let rows = test_file_reader_rows("repeated_no_annotation.parquet", None).unwrap();
- let expected_rows = vec![
- row![
- ("id".to_string(), Field::Int(1)),
- ("phoneNumbers".to_string(), Field::Null)
- ],
- row![
- ("id".to_string(), Field::Int(2)),
- ("phoneNumbers".to_string(), Field::Null)
- ],
- row![
- ("id".to_string(), Field::Int(3)),
- (
- "phoneNumbers".to_string(),
- group![("phone".to_string(), list![])]
- )
- ],
- row![
- ("id".to_string(), Field::Int(4)),
- (
- "phoneNumbers".to_string(),
- group![(
- "phone".to_string(),
- list![group![
- ("number".to_string(), Field::Long(5555555555)),
- ("kind".to_string(), Field::Null)
- ]]
- )]
- )
- ],
- row![
- ("id".to_string(), Field::Int(5)),
- (
- "phoneNumbers".to_string(),
- group![(
- "phone".to_string(),
- list![group![
- ("number".to_string(), Field::Long(1111111111)),
- ("kind".to_string(), Field::Str("home".to_string()))
- ]]
- )]
- )
- ],
- row![
- ("id".to_string(), Field::Int(6)),
- (
- "phoneNumbers".to_string(),
- group![(
- "phone".to_string(),
- list![
- group![
- ("number".to_string(), Field::Long(1111111111)),
- ("kind".to_string(), Field::Str("home".to_string()))
- ],
- group![
- ("number".to_string(), Field::Long(2222222222)),
- ("kind".to_string(), Field::Null)
- ],
- group![
- ("number".to_string(), Field::Long(3333333333)),
- ("kind".to_string(), Field::Str("mobile".to_string()))
- ]
- ]
- )]
- )
- ],
- ];
-
- assert_eq!(rows, expected_rows);
- }
-
- fn test_file_reader_rows(file_name: &str, schema: Option<Type>) -> Result<Vec<Row>> {
- let file = get_test_file(file_name);
- let file_reader: Box<dyn FileReader> = Box::new(SerializedFileReader::new(file)?);
- let iter = file_reader.get_row_iter(schema)?;
- Ok(iter.collect())
- }
-
- fn test_row_group_rows(file_name: &str, schema: Option<Type>) -> Result<Vec<Row>> {
- let file = get_test_file(file_name);
- let file_reader: Box<dyn FileReader> = Box::new(SerializedFileReader::new(file)?);
- // Check the first row group only, because files will contain only single row
- // group
- let row_group_reader = file_reader.get_row_group(0).unwrap();
- let iter = row_group_reader.get_row_iter(schema)?;
- Ok(iter.collect())
- }
-}
diff --git a/parquet/src/record/record_writer.rs b/parquet/src/record/record_writer.rs
deleted file mode 100644
index 56817eb..0000000
--- a/parquet/src/record/record_writer.rs
+++ /dev/null
@@ -1,26 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-use super::super::errors::ParquetError;
-use super::super::file::writer::RowGroupWriter;
-
-pub trait RecordWriter<T> {
- fn write_to_row_group(
- &self,
- row_group_writer: &mut Box<dyn RowGroupWriter>,
- ) -> Result<(), ParquetError>;
-}
diff --git a/parquet/src/record/triplet.rs b/parquet/src/record/triplet.rs
deleted file mode 100644
index bb4f942..0000000
--- a/parquet/src/record/triplet.rs
+++ /dev/null
@@ -1,561 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-use crate::basic::Type as PhysicalType;
-use crate::column::reader::{get_typed_column_reader, ColumnReader, ColumnReaderImpl};
-use crate::data_type::*;
-use crate::errors::{ParquetError, Result};
-use crate::record::api::Field;
-use crate::schema::types::ColumnDescPtr;
-
-/// Macro to generate simple functions that cover all types of triplet iterator.
-/// $func is a function of a typed triplet iterator and $token is a either {`ref`} or
-/// {`ref`, `mut`}
-macro_rules! triplet_enum_func {
- ($self:ident, $func:ident, $( $token:tt ),*) => ({
- match *$self {
- TripletIter::BoolTripletIter($($token)* typed) => typed.$func(),
- TripletIter::Int32TripletIter($($token)* typed) => typed.$func(),
- TripletIter::Int64TripletIter($($token)* typed) => typed.$func(),
- TripletIter::Int96TripletIter($($token)* typed) => typed.$func(),
- TripletIter::FloatTripletIter($($token)* typed) => typed.$func(),
- TripletIter::DoubleTripletIter($($token)* typed) => typed.$func(),
- TripletIter::ByteArrayTripletIter($($token)* typed) => typed.$func(),
- TripletIter::FixedLenByteArrayTripletIter($($token)* typed) => typed.$func()
- }
- });
-}
-
-/// High level API wrapper on column reader.
-/// Provides per-element access for each primitive column.
-pub enum TripletIter {
- BoolTripletIter(TypedTripletIter<BoolType>),
- Int32TripletIter(TypedTripletIter<Int32Type>),
- Int64TripletIter(TypedTripletIter<Int64Type>),
- Int96TripletIter(TypedTripletIter<Int96Type>),
- FloatTripletIter(TypedTripletIter<FloatType>),
- DoubleTripletIter(TypedTripletIter<DoubleType>),
- ByteArrayTripletIter(TypedTripletIter<ByteArrayType>),
- FixedLenByteArrayTripletIter(TypedTripletIter<FixedLenByteArrayType>),
-}
-
-impl TripletIter {
- /// Creates new triplet for column reader
- pub fn new(descr: ColumnDescPtr, reader: ColumnReader, batch_size: usize) -> Self {
- match descr.physical_type() {
- PhysicalType::BOOLEAN => TripletIter::BoolTripletIter(TypedTripletIter::new(
- descr, batch_size, reader,
- )),
- PhysicalType::INT32 => TripletIter::Int32TripletIter(TypedTripletIter::new(
- descr, batch_size, reader,
- )),
- PhysicalType::INT64 => TripletIter::Int64TripletIter(TypedTripletIter::new(
- descr, batch_size, reader,
- )),
- PhysicalType::INT96 => TripletIter::Int96TripletIter(TypedTripletIter::new(
- descr, batch_size, reader,
- )),
- PhysicalType::FLOAT => TripletIter::FloatTripletIter(TypedTripletIter::new(
- descr, batch_size, reader,
- )),
- PhysicalType::DOUBLE => TripletIter::DoubleTripletIter(
- TypedTripletIter::new(descr, batch_size, reader),
- ),
- PhysicalType::BYTE_ARRAY => TripletIter::ByteArrayTripletIter(
- TypedTripletIter::new(descr, batch_size, reader),
- ),
- PhysicalType::FIXED_LEN_BYTE_ARRAY => {
- TripletIter::FixedLenByteArrayTripletIter(TypedTripletIter::new(
- descr, batch_size, reader,
- ))
- }
- }
- }
-
- /// Invokes underlying typed triplet iterator to buffer current value.
- /// Should be called once - either before `is_null` or `current_value`.
- #[inline]
- pub fn read_next(&mut self) -> Result<bool> {
- triplet_enum_func!(self, read_next, ref, mut)
- }
-
- /// Provides check on values/levels left without invoking the underlying typed triplet
- /// iterator.
- /// Returns true if more values/levels exist, false otherwise.
- /// It is always in sync with `read_next` method.
- #[inline]
- pub fn has_next(&self) -> bool {
- triplet_enum_func!(self, has_next, ref)
- }
-
- /// Returns current definition level for a leaf triplet iterator
- #[inline]
- pub fn current_def_level(&self) -> i16 {
- triplet_enum_func!(self, current_def_level, ref)
- }
-
- /// Returns max definition level for a leaf triplet iterator
- #[inline]
- pub fn max_def_level(&self) -> i16 {
- triplet_enum_func!(self, max_def_level, ref)
- }
-
- /// Returns current repetition level for a leaf triplet iterator
- #[inline]
- pub fn current_rep_level(&self) -> i16 {
- triplet_enum_func!(self, current_rep_level, ref)
- }
-
- /// Returns max repetition level for a leaf triplet iterator
- #[inline]
- pub fn max_rep_level(&self) -> i16 {
- triplet_enum_func!(self, max_rep_level, ref)
- }
-
- /// Returns true, if current value is null.
- /// Based on the fact that for non-null value current definition level
- /// equals to max definition level.
- #[inline]
- pub fn is_null(&self) -> bool {
- self.current_def_level() < self.max_def_level()
- }
-
- /// Updates non-null value for current row.
- pub fn current_value(&self) -> Field {
- assert!(!self.is_null(), "Value is null");
- match *self {
- TripletIter::BoolTripletIter(ref typed) => {
- Field::convert_bool(typed.column_descr(), *typed.current_value())
- }
- TripletIter::Int32TripletIter(ref typed) => {
- Field::convert_int32(typed.column_descr(), *typed.current_value())
- }
- TripletIter::Int64TripletIter(ref typed) => {
- Field::convert_int64(typed.column_descr(), *typed.current_value())
- }
- TripletIter::Int96TripletIter(ref typed) => {
- Field::convert_int96(typed.column_descr(), typed.current_value().clone())
- }
- TripletIter::FloatTripletIter(ref typed) => {
- Field::convert_float(typed.column_descr(), *typed.current_value())
- }
- TripletIter::DoubleTripletIter(ref typed) => {
- Field::convert_double(typed.column_descr(), *typed.current_value())
- }
- TripletIter::ByteArrayTripletIter(ref typed) => Field::convert_byte_array(
- typed.column_descr(),
- typed.current_value().clone(),
- ),
- TripletIter::FixedLenByteArrayTripletIter(ref typed) => {
- Field::convert_byte_array(
- typed.column_descr(),
- typed.current_value().clone().into(),
- )
- }
- }
- }
-}
-
-/// Internal typed triplet iterator as a wrapper for column reader
-/// (primitive leaf column), provides per-element access.
-pub struct TypedTripletIter<T: DataType> {
- reader: ColumnReaderImpl<T>,
- column_descr: ColumnDescPtr,
- batch_size: usize,
- // type properties
- max_def_level: i16,
- max_rep_level: i16,
- // values and levels
- values: Vec<T::T>,
- def_levels: Option<Vec<i16>>,
- rep_levels: Option<Vec<i16>>,
- // current index for the triplet (value, def, rep)
- curr_triplet_index: usize,
- // how many triplets are left before we need to buffer
- triplets_left: usize,
- // helper flag to quickly check if we have more values/levels to read
- has_next: bool,
-}
-
-impl<T: DataType> TypedTripletIter<T> {
- /// Creates new typed triplet iterator based on provided column reader.
- /// Use batch size to specify the amount of values to buffer from column reader.
- fn new(descr: ColumnDescPtr, batch_size: usize, column_reader: ColumnReader) -> Self {
- assert!(
- batch_size > 0,
- "Expected positive batch size, found: {}",
- batch_size
- );
-
- let max_def_level = descr.max_def_level();
- let max_rep_level = descr.max_rep_level();
-
- let def_levels = if max_def_level == 0 {
- None
- } else {
- Some(vec![0; batch_size])
- };
- let rep_levels = if max_rep_level == 0 {
- None
- } else {
- Some(vec![0; batch_size])
- };
-
- Self {
- reader: get_typed_column_reader(column_reader),
- column_descr: descr,
- batch_size,
- max_def_level,
- max_rep_level,
- values: vec![T::T::default(); batch_size],
- def_levels,
- rep_levels,
- curr_triplet_index: 0,
- triplets_left: 0,
- has_next: false,
- }
- }
-
- /// Returns column descriptor reference for the current typed triplet iterator.
- #[inline]
- pub fn column_descr(&self) -> &ColumnDescPtr {
- &self.column_descr
- }
-
- /// Returns maximum definition level for the triplet iterator (leaf column).
- #[inline]
- fn max_def_level(&self) -> i16 {
- self.max_def_level
- }
-
- /// Returns maximum repetition level for the triplet iterator (leaf column).
- #[inline]
- fn max_rep_level(&self) -> i16 {
- self.max_rep_level
- }
-
- /// Returns current value.
- /// Method does not advance the iterator, therefore can be called multiple times.
- #[inline]
- fn current_value(&self) -> &T::T {
- assert!(
- self.current_def_level() == self.max_def_level(),
- "Cannot extract value, max definition level: {}, current level: {}",
- self.max_def_level(),
- self.current_def_level()
- );
- &self.values[self.curr_triplet_index]
- }
-
- /// Returns current definition level.
- /// If field is required, then maximum definition level is returned.
- #[inline]
- fn current_def_level(&self) -> i16 {
- match self.def_levels {
- Some(ref vec) => vec[self.curr_triplet_index],
- None => self.max_def_level,
- }
- }
-
- /// Returns current repetition level.
- /// If field is required, then maximum repetition level is returned.
- #[inline]
- fn current_rep_level(&self) -> i16 {
- match self.rep_levels {
- Some(ref vec) => vec[self.curr_triplet_index],
- None => self.max_rep_level,
- }
- }
-
- /// Quick check if iterator has more values/levels to read.
- /// It is updated as a result of `read_next` method, so they are synchronized.
- #[inline]
- fn has_next(&self) -> bool {
- self.has_next
- }
-
- /// Advances to the next triplet.
- /// Returns true, if there are more records to read, false there are no records left.
- fn read_next(&mut self) -> Result<bool> {
- self.curr_triplet_index += 1;
-
- if self.curr_triplet_index >= self.triplets_left {
- let (values_read, levels_read) = {
- // Get slice of definition levels, if available
- let def_levels = self.def_levels.as_mut().map(|vec| &mut vec[..]);
-
- // Get slice of repetition levels, if available
- let rep_levels = self.rep_levels.as_mut().map(|vec| &mut vec[..]);
-
- // Buffer triplets
- self.reader.read_batch(
- self.batch_size,
- def_levels,
- rep_levels,
- &mut self.values,
- )?
- };
-
- // No more values or levels to read
- if values_read == 0 && levels_read == 0 {
- self.has_next = false;
- return Ok(false);
- }
-
- // We never read values more than levels
- if levels_read == 0 || values_read == levels_read {
- // There are no definition levels to read, column is required
- // or definition levels match values, so it does not require spacing
- self.curr_triplet_index = 0;
- self.triplets_left = values_read;
- } else if values_read < levels_read {
- // Add spacing for triplets.
- // The idea is setting values for positions in def_levels when current
- // definition level equals to maximum definition level.
- // Values and levels are guaranteed to line up, because of
- // the column reader method.
-
- // Note: if values_read == 0, then spacing will not be triggered
- let mut idx = values_read;
- let def_levels = self.def_levels.as_ref().unwrap();
- for i in 0..levels_read {
- if def_levels[levels_read - i - 1] == self.max_def_level {
- idx -= 1; // This is done to avoid usize becoming a negative value
- self.values.swap(levels_read - i - 1, idx);
- }
- }
- self.curr_triplet_index = 0;
- self.triplets_left = levels_read;
- } else {
- return Err(general_err!(
- "Spacing of values/levels is wrong, values_read: {}, levels_read: {}",
- values_read,
- levels_read
- ));
- }
- }
-
- self.has_next = true;
- Ok(true)
- }
-}
-
-#[cfg(test)]
-mod tests {
- use super::*;
-
- use crate::file::reader::{FileReader, SerializedFileReader};
- use crate::schema::types::ColumnPath;
- use crate::util::test_common::get_test_file;
-
- #[test]
- #[should_panic(expected = "Expected positive batch size, found: 0")]
- fn test_triplet_zero_batch_size() {
- let column_path =
- ColumnPath::from(vec!["b_struct".to_string(), "b_c_int".to_string()]);
- test_column_in_file("nulls.snappy.parquet", 0, &column_path, &[], &[], &[]);
- }
-
- #[test]
- fn test_triplet_null_column() {
- let path = vec!["b_struct", "b_c_int"];
- let values = vec![];
- let def_levels = vec![1, 1, 1, 1, 1, 1, 1, 1];
- let rep_levels = vec![0, 0, 0, 0, 0, 0, 0, 0];
- test_triplet_iter(
- "nulls.snappy.parquet",
- path,
- &values,
- &def_levels,
- &rep_levels,
- );
- }
-
- #[test]
- fn test_triplet_required_column() {
- let path = vec!["ID"];
- let values = vec![Field::Long(8)];
- let def_levels = vec![0];
- let rep_levels = vec![0];
- test_triplet_iter(
- "nonnullable.impala.parquet",
- path,
- &values,
- &def_levels,
- &rep_levels,
- );
- }
-
- #[test]
- fn test_triplet_optional_column() {
- let path = vec!["nested_struct", "A"];
- let values = vec![Field::Int(1), Field::Int(7)];
- let def_levels = vec![2, 1, 1, 1, 1, 0, 2];
- let rep_levels = vec![0, 0, 0, 0, 0, 0, 0];
- test_triplet_iter(
- "nullable.impala.parquet",
- path,
- &values,
- &def_levels,
- &rep_levels,
- );
- }
-
- #[test]
- fn test_triplet_optional_list_column() {
- let path = vec!["a", "list", "element", "list", "element", "list", "element"];
- let values = vec![
- Field::Str("a".to_string()),
- Field::Str("b".to_string()),
- Field::Str("c".to_string()),
- Field::Str("d".to_string()),
- Field::Str("a".to_string()),
- Field::Str("b".to_string()),
- Field::Str("c".to_string()),
- Field::Str("d".to_string()),
- Field::Str("e".to_string()),
- Field::Str("a".to_string()),
- Field::Str("b".to_string()),
- Field::Str("c".to_string()),
- Field::Str("d".to_string()),
- Field::Str("e".to_string()),
- Field::Str("f".to_string()),
- ];
- let def_levels = vec![7, 7, 7, 4, 7, 7, 7, 7, 7, 4, 7, 7, 7, 7, 7, 7, 4, 7];
- let rep_levels = vec![0, 3, 2, 1, 2, 0, 3, 2, 3, 1, 2, 0, 3, 2, 3, 2, 1, 2];
- test_triplet_iter(
- "nested_lists.snappy.parquet",
- path,
- &values,
- &def_levels,
- &rep_levels,
- );
- }
-
- #[test]
- fn test_triplet_optional_map_column() {
- let path = vec!["a", "key_value", "value", "key_value", "key"];
- let values = vec![
- Field::Int(1),
- Field::Int(2),
- Field::Int(1),
- Field::Int(1),
- Field::Int(3),
- Field::Int(4),
- Field::Int(5),
- ];
- let def_levels = vec![4, 4, 4, 2, 3, 4, 4, 4, 4];
- let rep_levels = vec![0, 2, 0, 0, 0, 0, 0, 2, 2];
- test_triplet_iter(
- "nested_maps.snappy.parquet",
- path,
- &values,
- &def_levels,
- &rep_levels,
- );
- }
-
- // Check triplet iterator across different batch sizes
- fn test_triplet_iter(
- file_name: &str,
- column_path: Vec<&str>,
- expected_values: &[Field],
- expected_def_levels: &[i16],
- expected_rep_levels: &[i16],
- ) {
- // Convert path into column path
- let path: Vec<String> = column_path.iter().map(|x| x.to_string()).collect();
- let column_path = ColumnPath::from(path);
-
- let batch_sizes = vec![1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 128, 256];
- for batch_size in batch_sizes {
- test_column_in_file(
- file_name,
- batch_size,
- &column_path,
- expected_values,
- expected_def_levels,
- expected_rep_levels,
- );
- }
- }
-
- // Check values of a selectd column in a file
- fn test_column_in_file(
- file_name: &str,
- batch_size: usize,
- column_path: &ColumnPath,
- expected_values: &[Field],
- expected_def_levels: &[i16],
- expected_rep_levels: &[i16],
- ) {
- let file = get_test_file(file_name);
- let file_reader = SerializedFileReader::new(file).unwrap();
- let metadata = file_reader.metadata();
- // Get schema descriptor
- let file_metadata = metadata.file_metadata();
- let schema = file_metadata.schema_descr();
- // Get first row group
- let row_group_reader = file_reader.get_row_group(0).unwrap();
-
- for i in 0..schema.num_columns() {
- let descr = schema.column(i);
- if descr.path() == column_path {
- let reader = row_group_reader.get_column_reader(i).unwrap();
- test_triplet_column(
- descr,
- reader,
- batch_size,
- expected_values,
- expected_def_levels,
- expected_rep_levels,
- );
- }
- }
- }
-
- // Check values for individual triplet iterator
- fn test_triplet_column(
- descr: ColumnDescPtr,
- reader: ColumnReader,
- batch_size: usize,
- expected_values: &[Field],
- expected_def_levels: &[i16],
- expected_rep_levels: &[i16],
- ) {
- let mut iter = TripletIter::new(descr.clone(), reader, batch_size);
- let mut values: Vec<Field> = Vec::new();
- let mut def_levels: Vec<i16> = Vec::new();
- let mut rep_levels: Vec<i16> = Vec::new();
-
- assert_eq!(iter.max_def_level(), descr.max_def_level());
- assert_eq!(iter.max_rep_level(), descr.max_rep_level());
-
- while let Ok(true) = iter.read_next() {
- assert!(iter.has_next());
- if !iter.is_null() {
- values.push(iter.current_value());
- }
- def_levels.push(iter.current_def_level());
- rep_levels.push(iter.current_rep_level());
- }
-
- assert_eq!(values, expected_values);
- assert_eq!(def_levels, expected_def_levels);
- assert_eq!(rep_levels, expected_rep_levels);
- }
-}
diff --git a/parquet/src/schema/mod.rs b/parquet/src/schema/mod.rs
deleted file mode 100644
index 1ebee2e..0000000
--- a/parquet/src/schema/mod.rs
+++ /dev/null
@@ -1,67 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-//! Parquet schema definitions and methods to print and parse schema.
-//!
-//! # Example
-//!
-//! ```rust
-//! use parquet::{
-//! basic::{ConvertedType, Repetition, Type as PhysicalType},
-//! schema::{parser, printer, types::Type},
-//! };
-//! use std::sync::Arc;
-//!
-//! // Create the following schema:
-//! //
-//! // message schema {
-//! // OPTIONAL BYTE_ARRAY a (UTF8);
-//! // REQUIRED INT32 b;
-//! // }
-//!
-//! let field_a = Type::primitive_type_builder("a", PhysicalType::BYTE_ARRAY)
-//! .with_converted_type(ConvertedType::UTF8)
-//! .with_repetition(Repetition::OPTIONAL)
-//! .build()
-//! .unwrap();
-//!
-//! let field_b = Type::primitive_type_builder("b", PhysicalType::INT32)
-//! .with_repetition(Repetition::REQUIRED)
-//! .build()
-//! .unwrap();
-//!
-//! let schema = Type::group_type_builder("schema")
-//! .with_fields(&mut vec![Arc::new(field_a), Arc::new(field_b)])
-//! .build()
-//! .unwrap();
-//!
-//! let mut buf = Vec::new();
-//!
-//! // Print schema into buffer
-//! printer::print_schema(&mut buf, &schema);
-//!
-//! // Parse schema from the string
-//! let string_schema = String::from_utf8(buf).unwrap();
-//! let parsed_schema = parser::parse_message_type(&string_schema).unwrap();
-//!
-//! assert_eq!(schema, parsed_schema);
-//! ```
-
-pub mod parser;
-pub mod printer;
-pub mod types;
-pub mod visitor;
diff --git a/parquet/src/schema/parser.rs b/parquet/src/schema/parser.rs
deleted file mode 100644
index 41f6290..0000000
--- a/parquet/src/schema/parser.rs
+++ /dev/null
@@ -1,1241 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-//! Parquet schema parser.
-//! Provides methods to parse and validate string message type into Parquet
-//! [`Type`](crate::schema::types::Type).
-//!
-//! # Example
-//!
-//! ```rust
-//! use parquet::schema::parser::parse_message_type;
-//!
-//! let message_type = "
-//! message spark_schema {
-//! OPTIONAL BYTE_ARRAY a (UTF8);
-//! REQUIRED INT32 b;
-//! REQUIRED DOUBLE c;
-//! REQUIRED BOOLEAN d;
-//! OPTIONAL group e (LIST) {
-//! REPEATED group list {
-//! REQUIRED INT32 element;
-//! }
-//! }
-//! }
-//! ";
-//!
-//! let schema = parse_message_type(message_type).expect("Expected valid schema");
-//! println!("{:?}", schema);
-//! ```
-
-use std::sync::Arc;
-
-use crate::basic::{
- ConvertedType, DecimalType, IntType, LogicalType, Repetition, TimeType, TimeUnit,
- TimestampType, Type as PhysicalType,
-};
-use crate::errors::{ParquetError, Result};
-use crate::schema::types::{Type, TypePtr};
-
-/// Parses message type as string into a Parquet [`Type`](crate::schema::types::Type)
-/// which, for example, could be used to extract individual columns. Returns Parquet
-/// general error when parsing or validation fails.
-pub fn parse_message_type(message_type: &str) -> Result<Type> {
- let mut parser = Parser {
- tokenizer: &mut Tokenizer::from_str(message_type),
- };
- parser.parse_message_type()
-}
-
-/// Tokenizer to split message type string into tokens that are separated using characters
-/// defined in `is_schema_delim` method. Tokenizer also preserves delimiters as tokens.
-/// Tokenizer provides Iterator interface to process tokens; it also allows to step back
-/// to reprocess previous tokens.
-struct Tokenizer<'a> {
- // List of all tokens for a string
- tokens: Vec<&'a str>,
- // Current index of vector
- index: usize,
-}
-
-impl<'a> Tokenizer<'a> {
- // Create tokenizer from message type string
- pub fn from_str(string: &'a str) -> Self {
- let vec = string
- .split_whitespace()
- .flat_map(|t| Self::split_token(t))
- .collect();
- Tokenizer {
- tokens: vec,
- index: 0,
- }
- }
-
- // List of all special characters in schema
- fn is_schema_delim(c: char) -> bool {
- c == ';' || c == '{' || c == '}' || c == '(' || c == ')' || c == '=' || c == ','
- }
-
- /// Splits string into tokens; input string can already be token or can contain
- /// delimiters, e.g. required" -> Vec("required") and
- /// "(UTF8);" -> Vec("(", "UTF8", ")", ";")
- fn split_token(string: &str) -> Vec<&str> {
- let mut buffer: Vec<&str> = Vec::new();
- let mut tail = string;
- while let Some(index) = tail.find(Self::is_schema_delim) {
- let (h, t) = tail.split_at(index);
- if !h.is_empty() {
- buffer.push(h);
- }
- buffer.push(&t[0..1]);
- tail = &t[1..];
- }
- if !tail.is_empty() {
- buffer.push(tail);
- }
- buffer
- }
-
- // Move pointer to a previous element
- fn backtrack(&mut self) {
- self.index -= 1;
- }
-}
-
-impl<'a> Iterator for Tokenizer<'a> {
- type Item = &'a str;
-
- fn next(&mut self) -> Option<&'a str> {
- if self.index < self.tokens.len() {
- self.index += 1;
- Some(self.tokens[self.index - 1])
- } else {
- None
- }
- }
-}
-
-/// Internal Schema parser.
-/// Traverses message type using tokenizer and parses each group/primitive type
-/// recursively.
-struct Parser<'a> {
- tokenizer: &'a mut Tokenizer<'a>,
-}
-
-// Utility function to assert token on validity.
-fn assert_token(token: Option<&str>, expected: &str) -> Result<()> {
- match token {
- Some(value) if value == expected => Ok(()),
- Some(other) => Err(general_err!(
- "Expected '{}', found token '{}'",
- expected,
- other
- )),
- None => Err(general_err!(
- "Expected '{}', but no token found (None)",
- expected
- )),
- }
-}
-
-// Utility function to parse i32 or return general error.
-#[inline]
-fn parse_i32(
- value: Option<&str>,
- not_found_msg: &str,
- parse_fail_msg: &str,
-) -> Result<i32> {
- value
- .ok_or_else(|| general_err!(not_found_msg))
- .and_then(|v| v.parse::<i32>().map_err(|_| general_err!(parse_fail_msg)))
-}
-
-// Utility function to parse boolean or return general error.
-#[inline]
-fn parse_bool(
- value: Option<&str>,
- not_found_msg: &str,
- parse_fail_msg: &str,
-) -> Result<bool> {
- value
- .ok_or_else(|| general_err!(not_found_msg))
- .and_then(|v| {
- v.to_lowercase()
- .parse::<bool>()
- .map_err(|_| general_err!(parse_fail_msg))
- })
-}
-
-// Utility function to parse TimeUnit or return general error.
-fn parse_timeunit(
- value: Option<&str>,
- not_found_msg: &str,
- parse_fail_msg: &str,
-) -> Result<TimeUnit> {
- value
- .ok_or_else(|| general_err!(not_found_msg))
- .and_then(|v| match v.to_uppercase().as_str() {
- "MILLIS" => Ok(TimeUnit::MILLIS(Default::default())),
- "MICROS" => Ok(TimeUnit::MICROS(Default::default())),
- "NANOS" => Ok(TimeUnit::NANOS(Default::default())),
- _ => Err(general_err!(parse_fail_msg)),
- })
-}
-
-impl<'a> Parser<'a> {
- // Entry function to parse message type, uses internal tokenizer.
- fn parse_message_type(&mut self) -> Result<Type> {
- // Check that message type starts with "message".
- match self.tokenizer.next() {
- Some("message") => {
- let name = self
- .tokenizer
- .next()
- .ok_or_else(|| general_err!("Expected name, found None"))?;
- let mut fields = self.parse_child_types()?;
- Type::group_type_builder(name)
- .with_fields(&mut fields)
- .build()
- }
- _ => Err(general_err!("Message type does not start with 'message'")),
- }
- }
-
- // Parses child types for a current group type.
- // This is only invoked on root and group types.
- fn parse_child_types(&mut self) -> Result<Vec<TypePtr>> {
- assert_token(self.tokenizer.next(), "{")?;
- let mut vec = Vec::new();
- while let Some(value) = self.tokenizer.next() {
- if value == "}" {
- break;
- } else {
- self.tokenizer.backtrack();
- vec.push(Arc::new(self.add_type()?));
- }
- }
- Ok(vec)
- }
-
- fn add_type(&mut self) -> Result<Type> {
- // Parse repetition
- let repetition = self
- .tokenizer
- .next()
- .ok_or_else(|| general_err!("Expected repetition, found None"))
- .and_then(|v| v.to_uppercase().parse::<Repetition>())?;
-
- match self.tokenizer.next() {
- Some(group) if group.to_uppercase() == "GROUP" => {
- self.add_group_type(Some(repetition))
- }
- Some(type_string) => {
- let physical_type = type_string.to_uppercase().parse::<PhysicalType>()?;
- self.add_primitive_type(repetition, physical_type)
- }
- None => Err(general_err!("Invalid type, could not extract next token")),
- }
- }
-
- fn add_group_type(&mut self, repetition: Option<Repetition>) -> Result<Type> {
- // Parse name of the group type
- let name = self
- .tokenizer
- .next()
- .ok_or_else(|| general_err!("Expected name, found None"))?;
-
- // Parse logical or converted type if exists
- let (logical_type, converted_type) = if let Some("(") = self.tokenizer.next() {
- let tpe = self
- .tokenizer
- .next()
- .ok_or_else(|| general_err!("Expected converted type, found None"))
- .and_then(|v| {
- // Try logical type first
- let upper = v.to_uppercase();
- let logical = upper.parse::<LogicalType>();
- match logical {
- Ok(logical) => Ok((
- Some(logical.clone()),
- ConvertedType::from(Some(logical)),
- )),
- Err(_) => Ok((None, upper.parse::<ConvertedType>()?)),
- }
- })?;
- assert_token(self.tokenizer.next(), ")")?;
- tpe
- } else {
- self.tokenizer.backtrack();
- (None, ConvertedType::NONE)
- };
-
- // Parse optional id
- let id = if let Some("=") = self.tokenizer.next() {
- self.tokenizer.next().and_then(|v| v.parse::<i32>().ok())
- } else {
- self.tokenizer.backtrack();
- None
- };
-
- let mut fields = self.parse_child_types()?;
- let mut builder = Type::group_type_builder(name)
- .with_logical_type(logical_type)
- .with_converted_type(converted_type)
- .with_fields(&mut fields);
- if let Some(rep) = repetition {
- builder = builder.with_repetition(rep);
- }
- if let Some(id) = id {
- builder = builder.with_id(id);
- }
- builder.build()
- }
-
- fn add_primitive_type(
- &mut self,
- repetition: Repetition,
- physical_type: PhysicalType,
- ) -> Result<Type> {
- // Read type length if the type is FIXED_LEN_BYTE_ARRAY.
- let mut length: i32 = -1;
- if physical_type == PhysicalType::FIXED_LEN_BYTE_ARRAY {
- assert_token(self.tokenizer.next(), "(")?;
- length = parse_i32(
- self.tokenizer.next(),
- "Expected length for FIXED_LEN_BYTE_ARRAY, found None",
- "Failed to parse length for FIXED_LEN_BYTE_ARRAY",
- )?;
- assert_token(self.tokenizer.next(), ")")?;
- }
-
- // Parse name of the primitive type
- let name = self
- .tokenizer
- .next()
- .ok_or_else(|| general_err!("Expected name, found None"))?;
-
- // Parse converted type
- let (logical_type, converted_type, precision, scale) = if let Some("(") =
- self.tokenizer.next()
- {
- let (mut logical, mut converted) = self
- .tokenizer
- .next()
- .ok_or_else(|| {
- general_err!("Expected logical or converted type, found None")
- })
- .and_then(|v| {
- let upper = v.to_uppercase();
- let logical = upper.parse::<LogicalType>();
- match logical {
- Ok(logical) => Ok((
- Some(logical.clone()),
- ConvertedType::from(Some(logical)),
- )),
- Err(_) => Ok((None, upper.parse::<ConvertedType>()?)),
- }
- })?;
-
- // Parse precision and scale for decimals
- let mut precision: i32 = -1;
- let mut scale: i32 = -1;
-
- // Parse the concrete logical type
- if let Some(tpe) = &logical {
- match tpe {
- LogicalType::DECIMAL(_) => {
- if let Some("(") = self.tokenizer.next() {
- precision = parse_i32(
- self.tokenizer.next(),
- "Expected precision, found None",
- "Failed to parse precision for DECIMAL type",
- )?;
- if let Some(",") = self.tokenizer.next() {
- scale = parse_i32(
- self.tokenizer.next(),
- "Expected scale, found None",
- "Failed to parse scale for DECIMAL type",
- )?;
- assert_token(self.tokenizer.next(), ")")?;
- logical = Some(LogicalType::DECIMAL(DecimalType {
- scale,
- precision,
- }));
- converted = ConvertedType::from(logical.clone());
- } else {
- scale = 0;
- logical = Some(LogicalType::DECIMAL(DecimalType {
- scale,
- precision,
- }));
- converted = ConvertedType::from(logical.clone());
- }
- }
- }
- LogicalType::TIME(_) => {
- if let Some("(") = self.tokenizer.next() {
- let unit = parse_timeunit(
- self.tokenizer.next(),
- "Invalid timeunit found",
- "Failed to parse timeunit for TIME type",
- )?;
- if let Some(",") = self.tokenizer.next() {
- let is_adjusted_to_u_t_c = parse_bool(
- self.tokenizer.next(),
- "Invalid boolean found",
- "Failed to parse timezone info for TIME type",
- )?;
- assert_token(self.tokenizer.next(), ")")?;
- logical = Some(LogicalType::TIME(TimeType {
- is_adjusted_to_u_t_c,
- unit,
- }));
- converted = ConvertedType::from(logical.clone());
- } else {
- // Invalid token for unit
- self.tokenizer.backtrack();
- }
- }
- }
- LogicalType::TIMESTAMP(_) => {
- if let Some("(") = self.tokenizer.next() {
- let unit = parse_timeunit(
- self.tokenizer.next(),
- "Invalid timeunit found",
- "Failed to parse timeunit for TIMESTAMP type",
- )?;
- if let Some(",") = self.tokenizer.next() {
- let is_adjusted_to_u_t_c = parse_bool(
- self.tokenizer.next(),
- "Invalid boolean found",
- "Failed to parse timezone info for TIMESTAMP type",
- )?;
- assert_token(self.tokenizer.next(), ")")?;
- logical = Some(LogicalType::TIMESTAMP(TimestampType {
- is_adjusted_to_u_t_c,
- unit,
- }));
- converted = ConvertedType::from(logical.clone());
- } else {
- // Invalid token for unit
- self.tokenizer.backtrack();
- }
- }
- }
- LogicalType::INTEGER(_) => {
- if let Some("(") = self.tokenizer.next() {
- let bit_width = parse_i32(
- self.tokenizer.next(),
- "Invalid bit_width found",
- "Failed to parse bit_width for INTEGER type",
- )? as i8;
- match physical_type {
- PhysicalType::INT32 => {
- match bit_width {
- 8 | 16 | 32 => {}
- _ => {
- return Err(general_err!("Incorrect bit width {} for INT32", bit_width))
- }
- }
- }
- PhysicalType::INT64 => {
- if bit_width != 64 {
- return Err(general_err!("Incorrect bit width {} for INT64", bit_width))
- }
- }
- _ => {
- return Err(general_err!("Logical type INTEGER cannot be used with physical type {}", physical_type))
- }
- }
- if let Some(",") = self.tokenizer.next() {
- let is_signed = parse_bool(
- self.tokenizer.next(),
- "Invalid boolean found",
- "Failed to parse is_signed for INTEGER type",
- )?;
- assert_token(self.tokenizer.next(), ")")?;
- logical = Some(LogicalType::INTEGER(IntType {
- bit_width,
- is_signed,
- }));
- converted = ConvertedType::from(logical.clone());
- } else {
- // Invalid token for unit
- self.tokenizer.backtrack();
- }
- }
- }
- _ => {}
- }
- } else if converted == ConvertedType::DECIMAL {
- if let Some("(") = self.tokenizer.next() {
- // Parse precision
- precision = parse_i32(
- self.tokenizer.next(),
- "Expected precision, found None",
- "Failed to parse precision for DECIMAL type",
- )?;
-
- // Parse scale
- scale = if let Some(",") = self.tokenizer.next() {
- parse_i32(
- self.tokenizer.next(),
- "Expected scale, found None",
- "Failed to parse scale for DECIMAL type",
- )?
- } else {
- // Scale is not provided, set it to 0.
- self.tokenizer.backtrack();
- 0
- };
-
- assert_token(self.tokenizer.next(), ")")?;
- } else {
- self.tokenizer.backtrack();
- }
- }
-
- assert_token(self.tokenizer.next(), ")")?;
- (logical, converted, precision, scale)
- } else {
- self.tokenizer.backtrack();
- (None, ConvertedType::NONE, -1, -1)
- };
-
- // Parse optional id
- let id = if let Some("=") = self.tokenizer.next() {
- self.tokenizer.next().and_then(|v| v.parse::<i32>().ok())
- } else {
- self.tokenizer.backtrack();
- None
- };
- assert_token(self.tokenizer.next(), ";")?;
-
- let mut builder = Type::primitive_type_builder(name, physical_type)
- .with_repetition(repetition)
- .with_logical_type(logical_type)
- .with_converted_type(converted_type)
- .with_length(length)
- .with_precision(precision)
- .with_scale(scale);
- if let Some(id) = id {
- builder = builder.with_id(id);
- }
- builder.build()
- }
-}
-
-#[cfg(test)]
-mod tests {
- use super::*;
-
- #[test]
- fn test_tokenize_empty_string() {
- assert_eq!(Tokenizer::from_str("").next(), None);
- }
-
- #[test]
- fn test_tokenize_delimiters() {
- let mut iter = Tokenizer::from_str(",;{}()=");
- assert_eq!(iter.next(), Some(","));
- assert_eq!(iter.next(), Some(";"));
- assert_eq!(iter.next(), Some("{"));
- assert_eq!(iter.next(), Some("}"));
- assert_eq!(iter.next(), Some("("));
- assert_eq!(iter.next(), Some(")"));
- assert_eq!(iter.next(), Some("="));
- assert_eq!(iter.next(), None);
- }
-
- #[test]
- fn test_tokenize_delimiters_with_whitespaces() {
- let mut iter = Tokenizer::from_str(" , ; { } ( ) = ");
- assert_eq!(iter.next(), Some(","));
- assert_eq!(iter.next(), Some(";"));
- assert_eq!(iter.next(), Some("{"));
- assert_eq!(iter.next(), Some("}"));
- assert_eq!(iter.next(), Some("("));
- assert_eq!(iter.next(), Some(")"));
- assert_eq!(iter.next(), Some("="));
- assert_eq!(iter.next(), None);
- }
-
- #[test]
- fn test_tokenize_words() {
- let mut iter = Tokenizer::from_str("abc def ghi jkl mno");
- assert_eq!(iter.next(), Some("abc"));
- assert_eq!(iter.next(), Some("def"));
- assert_eq!(iter.next(), Some("ghi"));
- assert_eq!(iter.next(), Some("jkl"));
- assert_eq!(iter.next(), Some("mno"));
- assert_eq!(iter.next(), None);
- }
-
- #[test]
- fn test_tokenize_backtrack() {
- let mut iter = Tokenizer::from_str("abc;");
- assert_eq!(iter.next(), Some("abc"));
- assert_eq!(iter.next(), Some(";"));
- iter.backtrack();
- assert_eq!(iter.next(), Some(";"));
- assert_eq!(iter.next(), None);
- }
-
- #[test]
- fn test_tokenize_message_type() {
- let schema = "
- message schema {
- required int32 a;
- optional binary c (UTF8);
- required group d {
- required int32 a;
- optional binary c (UTF8);
- }
- required group e (LIST) {
- repeated group list {
- required int32 element;
- }
- }
- }
- ";
- let iter = Tokenizer::from_str(schema);
- let mut res = Vec::new();
- for token in iter {
- res.push(token);
- }
- assert_eq!(
- res,
- vec![
- "message", "schema", "{", "required", "int32", "a", ";", "optional",
- "binary", "c", "(", "UTF8", ")", ";", "required", "group", "d", "{",
- "required", "int32", "a", ";", "optional", "binary", "c", "(", "UTF8",
- ")", ";", "}", "required", "group", "e", "(", "LIST", ")", "{",
- "repeated", "group", "list", "{", "required", "int32", "element", ";",
- "}", "}", "}"
- ]
- );
- }
-
- #[test]
- fn test_assert_token() {
- assert!(assert_token(Some("a"), "a").is_ok());
- assert!(assert_token(Some("a"), "b").is_err());
- assert!(assert_token(None, "b").is_err());
- }
-
- #[test]
- fn test_parse_message_type_invalid() {
- let mut iter = Tokenizer::from_str("test");
- let result = Parser {
- tokenizer: &mut iter,
- }
- .parse_message_type();
- assert!(result.is_err());
- assert_eq!(
- result.unwrap_err().to_string(),
- "Parquet error: Message type does not start with 'message'"
- );
- }
-
- #[test]
- fn test_parse_message_type_no_name() {
- let mut iter = Tokenizer::from_str("message");
- let result = Parser {
- tokenizer: &mut iter,
- }
- .parse_message_type();
- assert!(result.is_err());
- assert_eq!(
- result.unwrap_err().to_string(),
- "Parquet error: Expected name, found None"
- );
- }
-
- #[test]
- fn test_parse_message_type_fixed_byte_array() {
- let schema = "
- message schema {
- REQUIRED FIXED_LEN_BYTE_ARRAY col;
- }
- ";
- let mut iter = Tokenizer::from_str(schema);
- let result = Parser {
- tokenizer: &mut iter,
- }
- .parse_message_type();
- assert!(result.is_err());
-
- let schema = "
- message schema {
- REQUIRED FIXED_LEN_BYTE_ARRAY(16) col;
- }
- ";
- let mut iter = Tokenizer::from_str(schema);
- let result = Parser {
- tokenizer: &mut iter,
- }
- .parse_message_type();
- assert!(result.is_ok());
- }
-
- #[test]
- fn test_parse_message_type_integer() {
- // Invalid integer syntax
- let schema = "
- message root {
- optional int64 f1 (INTEGER());
- }
- ";
- let mut iter = Tokenizer::from_str(schema);
- let result = Parser {
- tokenizer: &mut iter,
- }
- .parse_message_type();
- assert_eq!(
- result,
- Err(general_err!("Failed to parse bit_width for INTEGER type"))
- );
-
- // Invalid integer syntax, needs both bit-width and UTC sign
- let schema = "
- message root {
- optional int64 f1 (INTEGER(32,));
- }
- ";
- let mut iter = Tokenizer::from_str(schema);
- let result = Parser {
- tokenizer: &mut iter,
- }
- .parse_message_type();
- assert_eq!(
- result,
- Err(general_err!("Incorrect bit width 32 for INT64"))
- );
-
- // Invalid integer because of non-numeric bit width
- let schema = "
- message root {
- optional int32 f1 (INTEGER(eight,true));
- }
- ";
- let mut iter = Tokenizer::from_str(schema);
- let result = Parser {
- tokenizer: &mut iter,
- }
- .parse_message_type();
- assert_eq!(
- result,
- Err(general_err!("Failed to parse bit_width for INTEGER type"))
- );
-
- // Valid types
- let schema = "
- message root {
- optional int32 f1 (INTEGER(8,false));
- optional int32 f2 (INTEGER(8,true));
- optional int32 f3 (INTEGER(16,false));
- optional int32 f4 (INTEGER(16,true));
- optional int32 f5 (INTEGER(32,false));
- optional int32 f6 (INTEGER(32,true));
- optional int64 f7 (INTEGER(64,false));
- optional int64 f7 (INTEGER(64,true));
- }
- ";
- let mut iter = Tokenizer::from_str(schema);
- let result = Parser {
- tokenizer: &mut iter,
- }
- .parse_message_type();
- assert!(result.is_ok());
- }
-
- #[test]
- fn test_parse_message_type_temporal() {
- // Invalid timestamp syntax
- let schema = "
- message root {
- optional int64 f1 (TIMESTAMP();
- }
- ";
- let mut iter = Tokenizer::from_str(schema);
- let result = Parser {
- tokenizer: &mut iter,
- }
- .parse_message_type();
- assert_eq!(
- result,
- Err(general_err!("Failed to parse timeunit for TIMESTAMP type"))
- );
-
- // Invalid timestamp syntax, needs both unit and UTC adjustment
- let schema = "
- message root {
- optional int64 f1 (TIMESTAMP(MILLIS,));
- }
- ";
- let mut iter = Tokenizer::from_str(schema);
- let result = Parser {
- tokenizer: &mut iter,
- }
- .parse_message_type();
- assert_eq!(
- result,
- Err(general_err!(
- "Failed to parse timezone info for TIMESTAMP type"
- ))
- );
-
- // Invalid timestamp because of unknown unit
- let schema = "
- message root {
- optional int64 f1 (TIMESTAMP(YOCTOS,));
- }
- ";
- let mut iter = Tokenizer::from_str(schema);
- let result = Parser {
- tokenizer: &mut iter,
- }
- .parse_message_type();
- assert_eq!(
- result,
- Err(general_err!("Failed to parse timeunit for TIMESTAMP type"))
- );
-
- // Valid types
- let schema = "
- message root {
- optional int32 f1 (DATE);
- optional int32 f2 (TIME(MILLIS,true));
- optional int64 f3 (TIME(MICROS,false));
- optional int64 f4 (TIME(NANOS,true));
- optional int64 f5 (TIMESTAMP(MILLIS,true));
- optional int64 f6 (TIMESTAMP(MICROS,true));
- optional int64 f7 (TIMESTAMP(NANOS,false));
- }
- ";
- let mut iter = Tokenizer::from_str(schema);
- let result = Parser {
- tokenizer: &mut iter,
- }
- .parse_message_type();
- assert!(result.is_ok());
- }
-
- #[test]
- fn test_parse_message_type_decimal() {
- // It is okay for decimal to omit precision and scale with right syntax.
- // Here we test wrong syntax of decimal type
-
- // Invalid decimal syntax
- let schema = "
- message root {
- optional int32 f1 (DECIMAL();
- }
- ";
- let mut iter = Tokenizer::from_str(schema);
- let result = Parser {
- tokenizer: &mut iter,
- }
- .parse_message_type();
- assert!(result.is_err());
-
- // Invalid decimal, need precision and scale
- let schema = "
- message root {
- optional int32 f1 (DECIMAL());
- }
- ";
- let mut iter = Tokenizer::from_str(schema);
- let result = Parser {
- tokenizer: &mut iter,
- }
- .parse_message_type();
- assert!(result.is_err());
-
- // Invalid decimal because of `,` - has precision, needs scale
- let schema = "
- message root {
- optional int32 f1 (DECIMAL(8,));
- }
- ";
- let mut iter = Tokenizer::from_str(schema);
- let result = Parser {
- tokenizer: &mut iter,
- }
- .parse_message_type();
- assert!(result.is_err());
-
- // Invalid decimal because, we always require either precision or scale to be
- // specified as part of converted type
- let schema = "
- message root {
- optional int32 f3 (DECIMAL);
- }
- ";
- let mut iter = Tokenizer::from_str(schema);
- let result = Parser {
- tokenizer: &mut iter,
- }
- .parse_message_type();
- assert!(result.is_err());
-
- // Valid decimal (precision, scale)
- let schema = "
- message root {
- optional int32 f1 (DECIMAL(8, 3));
- optional int32 f2 (DECIMAL(8));
- }
- ";
- let mut iter = Tokenizer::from_str(schema);
- let result = Parser {
- tokenizer: &mut iter,
- }
- .parse_message_type();
- assert!(result.is_ok());
- }
-
- #[test]
- fn test_parse_message_type_compare_1() {
- let schema = "
- message root {
- optional fixed_len_byte_array(5) f1 (DECIMAL(9, 3));
- optional fixed_len_byte_array (16) f2 (DECIMAL (38, 18));
- }
- ";
- let mut iter = Tokenizer::from_str(schema);
- let message = Parser {
- tokenizer: &mut iter,
- }
- .parse_message_type()
- .unwrap();
-
- let expected = Type::group_type_builder("root")
- .with_fields(&mut vec![
- Arc::new(
- Type::primitive_type_builder(
- "f1",
- PhysicalType::FIXED_LEN_BYTE_ARRAY,
- )
- .with_logical_type(Some(LogicalType::DECIMAL(DecimalType {
- precision: 9,
- scale: 3,
- })))
- .with_converted_type(ConvertedType::DECIMAL)
- .with_length(5)
- .with_precision(9)
- .with_scale(3)
- .build()
- .unwrap(),
- ),
- Arc::new(
- Type::primitive_type_builder(
- "f2",
- PhysicalType::FIXED_LEN_BYTE_ARRAY,
- )
- .with_logical_type(Some(LogicalType::DECIMAL(DecimalType {
- precision: 38,
- scale: 18,
- })))
- .with_converted_type(ConvertedType::DECIMAL)
- .with_length(16)
- .with_precision(38)
- .with_scale(18)
- .build()
- .unwrap(),
- ),
- ])
- .build()
- .unwrap();
-
- assert_eq!(message, expected);
- }
-
- #[test]
- fn test_parse_message_type_compare_2() {
- let schema = "
- message root {
- required group a0 {
- optional group a1 (LIST) {
- repeated binary a2 (UTF8);
- }
-
- optional group b1 (LIST) {
- repeated group b2 {
- optional int32 b3;
- optional double b4;
- }
- }
- }
- }
- ";
- let mut iter = Tokenizer::from_str(schema);
- let message = Parser {
- tokenizer: &mut iter,
- }
- .parse_message_type()
- .unwrap();
-
- let expected = Type::group_type_builder("root")
- .with_fields(&mut vec![Arc::new(
- Type::group_type_builder("a0")
- .with_repetition(Repetition::REQUIRED)
- .with_fields(&mut vec![
- Arc::new(
- Type::group_type_builder("a1")
- .with_repetition(Repetition::OPTIONAL)
- .with_logical_type(Some(LogicalType::LIST(
- Default::default(),
- )))
- .with_converted_type(ConvertedType::LIST)
- .with_fields(&mut vec![Arc::new(
- Type::primitive_type_builder(
- "a2",
- PhysicalType::BYTE_ARRAY,
- )
- .with_repetition(Repetition::REPEATED)
- .with_converted_type(ConvertedType::UTF8)
- .build()
- .unwrap(),
- )])
- .build()
- .unwrap(),
- ),
- Arc::new(
- Type::group_type_builder("b1")
- .with_repetition(Repetition::OPTIONAL)
- .with_logical_type(Some(LogicalType::LIST(
- Default::default(),
- )))
- .with_converted_type(ConvertedType::LIST)
- .with_fields(&mut vec![Arc::new(
- Type::group_type_builder("b2")
- .with_repetition(Repetition::REPEATED)
- .with_fields(&mut vec![
- Arc::new(
- Type::primitive_type_builder(
- "b3",
- PhysicalType::INT32,
- )
- .build()
- .unwrap(),
- ),
- Arc::new(
- Type::primitive_type_builder(
- "b4",
- PhysicalType::DOUBLE,
- )
- .build()
- .unwrap(),
- ),
- ])
- .build()
- .unwrap(),
- )])
- .build()
- .unwrap(),
- ),
- ])
- .build()
- .unwrap(),
- )])
- .build()
- .unwrap();
-
- assert_eq!(message, expected);
- }
-
- #[test]
- fn test_parse_message_type_compare_3() {
- let schema = "
- message root {
- required int32 _1 (INT_8);
- required int32 _2 (INT_16);
- required float _3;
- required double _4;
- optional int32 _5 (DATE);
- optional binary _6 (UTF8);
- }
- ";
- let mut iter = Tokenizer::from_str(schema);
- let message = Parser {
- tokenizer: &mut iter,
- }
- .parse_message_type()
- .unwrap();
-
- let mut fields = vec![
- Arc::new(
- Type::primitive_type_builder("_1", PhysicalType::INT32)
- .with_repetition(Repetition::REQUIRED)
- .with_converted_type(ConvertedType::INT_8)
- .build()
- .unwrap(),
- ),
- Arc::new(
- Type::primitive_type_builder("_2", PhysicalType::INT32)
- .with_repetition(Repetition::REQUIRED)
- .with_converted_type(ConvertedType::INT_16)
- .build()
- .unwrap(),
- ),
- Arc::new(
- Type::primitive_type_builder("_3", PhysicalType::FLOAT)
- .with_repetition(Repetition::REQUIRED)
- .build()
- .unwrap(),
- ),
- Arc::new(
- Type::primitive_type_builder("_4", PhysicalType::DOUBLE)
- .with_repetition(Repetition::REQUIRED)
- .build()
- .unwrap(),
- ),
- Arc::new(
- Type::primitive_type_builder("_5", PhysicalType::INT32)
- .with_logical_type(Some(LogicalType::DATE(Default::default())))
- .with_converted_type(ConvertedType::DATE)
- .build()
- .unwrap(),
- ),
- Arc::new(
- Type::primitive_type_builder("_6", PhysicalType::BYTE_ARRAY)
- .with_converted_type(ConvertedType::UTF8)
- .build()
- .unwrap(),
- ),
- ];
-
- let expected = Type::group_type_builder("root")
- .with_fields(&mut fields)
- .build()
- .unwrap();
- assert_eq!(message, expected);
- }
-
- #[test]
- fn test_parse_message_type_compare_4() {
- let schema = "
- message root {
- required int32 _1 (INTEGER(8,true));
- required int32 _2 (INTEGER(16,false));
- required float _3;
- required double _4;
- optional int32 _5 (DATE);
- optional int32 _6 (TIME(MILLIS,false));
- optional int64 _7 (TIME(MICROS,true));
- optional int64 _8 (TIMESTAMP(MILLIS,true));
- optional int64 _9 (TIMESTAMP(NANOS,false));
- optional binary _10 (STRING);
- }
- ";
- let mut iter = Tokenizer::from_str(schema);
- let message = Parser {
- tokenizer: &mut iter,
- }
- .parse_message_type()
- .unwrap();
-
- let mut fields = vec![
- Arc::new(
- Type::primitive_type_builder("_1", PhysicalType::INT32)
- .with_repetition(Repetition::REQUIRED)
- .with_logical_type(Some(LogicalType::INTEGER(IntType {
- bit_width: 8,
- is_signed: true,
- })))
- .build()
- .unwrap(),
- ),
- Arc::new(
- Type::primitive_type_builder("_2", PhysicalType::INT32)
- .with_repetition(Repetition::REQUIRED)
- .with_logical_type(Some(LogicalType::INTEGER(IntType {
- bit_width: 16,
- is_signed: false,
- })))
- .build()
- .unwrap(),
- ),
- Arc::new(
- Type::primitive_type_builder("_3", PhysicalType::FLOAT)
- .with_repetition(Repetition::REQUIRED)
- .build()
- .unwrap(),
- ),
- Arc::new(
- Type::primitive_type_builder("_4", PhysicalType::DOUBLE)
- .with_repetition(Repetition::REQUIRED)
- .build()
- .unwrap(),
- ),
- Arc::new(
- Type::primitive_type_builder("_5", PhysicalType::INT32)
- .with_logical_type(Some(LogicalType::DATE(Default::default())))
- .build()
- .unwrap(),
- ),
- Arc::new(
- Type::primitive_type_builder("_6", PhysicalType::INT32)
- .with_logical_type(Some(LogicalType::TIME(TimeType {
- unit: TimeUnit::MILLIS(Default::default()),
- is_adjusted_to_u_t_c: false,
- })))
- .build()
- .unwrap(),
- ),
- Arc::new(
- Type::primitive_type_builder("_7", PhysicalType::INT64)
- .with_logical_type(Some(LogicalType::TIME(TimeType {
- unit: TimeUnit::MICROS(Default::default()),
- is_adjusted_to_u_t_c: true,
- })))
- .build()
- .unwrap(),
- ),
- Arc::new(
- Type::primitive_type_builder("_8", PhysicalType::INT64)
- .with_logical_type(Some(LogicalType::TIMESTAMP(TimestampType {
- unit: TimeUnit::MILLIS(Default::default()),
- is_adjusted_to_u_t_c: true,
- })))
- .build()
- .unwrap(),
- ),
- Arc::new(
- Type::primitive_type_builder("_9", PhysicalType::INT64)
- .with_logical_type(Some(LogicalType::TIMESTAMP(TimestampType {
- unit: TimeUnit::NANOS(Default::default()),
- is_adjusted_to_u_t_c: false,
- })))
- .build()
- .unwrap(),
- ),
- Arc::new(
- Type::primitive_type_builder("_10", PhysicalType::BYTE_ARRAY)
- .with_logical_type(Some(LogicalType::STRING(Default::default())))
- .build()
- .unwrap(),
- ),
- ];
-
- let expected = Type::group_type_builder("root")
- .with_fields(&mut fields)
- .build()
- .unwrap();
- assert_eq!(message, expected);
- }
-}
diff --git a/parquet/src/schema/printer.rs b/parquet/src/schema/printer.rs
deleted file mode 100644
index b1e739f..0000000
--- a/parquet/src/schema/printer.rs
+++ /dev/null
@@ -1,827 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-//! Parquet schema printer.
-//! Provides methods to print Parquet file schema and list file metadata.
-//!
-//! # Example
-//!
-//! ```rust
-//! use parquet::{
-//! file::reader::{FileReader, SerializedFileReader},
-//! schema::printer::{print_file_metadata, print_parquet_metadata, print_schema},
-//! };
-//! use std::{fs::File, path::Path};
-//!
-//! // Open a file
-//! let path = Path::new("test.parquet");
-//! if let Ok(file) = File::open(&path) {
-//! let reader = SerializedFileReader::new(file).unwrap();
-//! let parquet_metadata = reader.metadata();
-//!
-//! print_parquet_metadata(&mut std::io::stdout(), &parquet_metadata);
-//! print_file_metadata(&mut std::io::stdout(), &parquet_metadata.file_metadata());
-//!
-//! print_schema(
-//! &mut std::io::stdout(),
-//! &parquet_metadata.file_metadata().schema(),
-//! );
-//! }
-//! ```
-
-use std::{fmt, io};
-
-use crate::basic::{ConvertedType, LogicalType, TimeUnit, Type as PhysicalType};
-use crate::file::metadata::{
- ColumnChunkMetaData, FileMetaData, ParquetMetaData, RowGroupMetaData,
-};
-use crate::schema::types::Type;
-
-/// Prints Parquet metadata [`ParquetMetaData`](crate::file::metadata::ParquetMetaData)
-/// information.
-#[allow(unused_must_use)]
-pub fn print_parquet_metadata(out: &mut dyn io::Write, metadata: &ParquetMetaData) {
- print_file_metadata(out, &metadata.file_metadata());
- writeln!(out);
- writeln!(out);
- writeln!(out, "num of row groups: {}", metadata.num_row_groups());
- writeln!(out, "row groups:");
- writeln!(out);
- for (i, rg) in metadata.row_groups().iter().enumerate() {
- writeln!(out, "row group {}:", i);
- print_dashes(out, 80);
- print_row_group_metadata(out, rg);
- }
-}
-
-/// Prints file metadata [`FileMetaData`](crate::file::metadata::FileMetaData)
-/// information.
-#[allow(unused_must_use)]
-pub fn print_file_metadata(out: &mut dyn io::Write, file_metadata: &FileMetaData) {
- writeln!(out, "version: {}", file_metadata.version());
- writeln!(out, "num of rows: {}", file_metadata.num_rows());
- if let Some(created_by) = file_metadata.created_by().as_ref() {
- writeln!(out, "created by: {}", created_by);
- }
- if let Some(metadata) = file_metadata.key_value_metadata() {
- writeln!(out, "metadata:");
- for kv in metadata.iter() {
- writeln!(
- out,
- " {}: {}",
- &kv.key,
- kv.value.as_ref().unwrap_or(&"".to_owned())
- );
- }
- }
- let schema = file_metadata.schema();
- print_schema(out, schema);
-}
-
-/// Prints Parquet [`Type`](crate::schema::types::Type) information.
-#[allow(unused_must_use)]
-pub fn print_schema(out: &mut dyn io::Write, tp: &Type) {
- // TODO: better if we can pass fmt::Write to Printer.
- // But how can we make it to accept both io::Write & fmt::Write?
- let mut s = String::new();
- {
- let mut printer = Printer::new(&mut s);
- printer.print(tp);
- }
- writeln!(out, "{}", s);
-}
-
-#[allow(unused_must_use)]
-fn print_row_group_metadata(out: &mut dyn io::Write, rg_metadata: &RowGroupMetaData) {
- writeln!(out, "total byte size: {}", rg_metadata.total_byte_size());
- writeln!(out, "num of rows: {}", rg_metadata.num_rows());
- writeln!(out);
- writeln!(out, "num of columns: {}", rg_metadata.num_columns());
- writeln!(out, "columns: ");
- for (i, cc) in rg_metadata.columns().iter().enumerate() {
- writeln!(out);
- writeln!(out, "column {}:", i);
- print_dashes(out, 80);
- print_column_chunk_metadata(out, cc);
- }
-}
-
-#[allow(unused_must_use)]
-fn print_column_chunk_metadata(
- out: &mut dyn io::Write,
- cc_metadata: &ColumnChunkMetaData,
-) {
- writeln!(out, "column type: {}", cc_metadata.column_type());
- writeln!(out, "column path: {}", cc_metadata.column_path());
- let encoding_strs: Vec<_> = cc_metadata
- .encodings()
- .iter()
- .map(|e| format!("{}", e))
- .collect();
- writeln!(out, "encodings: {}", encoding_strs.join(" "));
- let file_path_str = match cc_metadata.file_path() {
- None => "N/A",
- Some(ref fp) => *fp,
- };
- writeln!(out, "file path: {}", file_path_str);
- writeln!(out, "file offset: {}", cc_metadata.file_offset());
- writeln!(out, "num of values: {}", cc_metadata.num_values());
- writeln!(
- out,
- "total compressed size (in bytes): {}",
- cc_metadata.compressed_size()
- );
- writeln!(
- out,
- "total uncompressed size (in bytes): {}",
- cc_metadata.uncompressed_size()
- );
- writeln!(out, "data page offset: {}", cc_metadata.data_page_offset());
- let index_page_offset_str = match cc_metadata.index_page_offset() {
- None => "N/A".to_owned(),
- Some(ipo) => ipo.to_string(),
- };
- writeln!(out, "index page offset: {}", index_page_offset_str);
- let dict_page_offset_str = match cc_metadata.dictionary_page_offset() {
- None => "N/A".to_owned(),
- Some(dpo) => dpo.to_string(),
- };
- writeln!(out, "dictionary page offset: {}", dict_page_offset_str);
- let statistics_str = match cc_metadata.statistics() {
- None => "N/A".to_owned(),
- Some(stats) => stats.to_string(),
- };
- writeln!(out, "statistics: {}", statistics_str);
- writeln!(out);
-}
-
-#[allow(unused_must_use)]
-fn print_dashes(out: &mut dyn io::Write, num: i32) {
- for _ in 0..num {
- write!(out, "-");
- }
- writeln!(out);
-}
-
-const INDENT_WIDTH: i32 = 2;
-
-/// Struct for printing Parquet message type.
-struct Printer<'a> {
- output: &'a mut dyn fmt::Write,
- indent: i32,
-}
-
-#[allow(unused_must_use)]
-impl<'a> Printer<'a> {
- fn new(output: &'a mut dyn fmt::Write) -> Self {
- Printer { output, indent: 0 }
- }
-
- fn print_indent(&mut self) {
- for _ in 0..self.indent {
- write!(self.output, " ");
- }
- }
-}
-
-#[inline]
-fn print_timeunit(unit: &TimeUnit) -> &str {
- match unit {
- TimeUnit::MILLIS(_) => "MILLIS",
- TimeUnit::MICROS(_) => "MICROS",
- TimeUnit::NANOS(_) => "NANOS",
- }
-}
-
-#[inline]
-fn print_logical_and_converted(
- logical_type: &Option<LogicalType>,
- converted_type: ConvertedType,
- precision: i32,
- scale: i32,
-) -> String {
- match logical_type {
- Some(logical_type) => match logical_type {
- LogicalType::INTEGER(t) => {
- format!("INTEGER({},{})", t.bit_width, t.is_signed)
- }
- LogicalType::DECIMAL(t) => {
- format!("DECIMAL({},{})", t.precision, t.scale)
- }
- LogicalType::TIMESTAMP(t) => {
- format!(
- "TIMESTAMP({},{})",
- print_timeunit(&t.unit),
- t.is_adjusted_to_u_t_c
- )
- }
- LogicalType::TIME(t) => {
- format!(
- "TIME({},{})",
- print_timeunit(&t.unit),
- t.is_adjusted_to_u_t_c
- )
- }
- LogicalType::DATE(_) => "DATE".to_string(),
- LogicalType::BSON(_) => "BSON".to_string(),
- LogicalType::JSON(_) => "JSON".to_string(),
- LogicalType::STRING(_) => "STRING".to_string(),
- LogicalType::UUID(_) => "UUID".to_string(),
- LogicalType::ENUM(_) => "ENUM".to_string(),
- LogicalType::LIST(_) => "LIST".to_string(),
- LogicalType::MAP(_) => "MAP".to_string(),
- LogicalType::UNKNOWN(_) => "UNKNOWN".to_string(),
- },
- None => {
- // Also print converted type if it is available
- match converted_type {
- ConvertedType::NONE => format!(""),
- decimal @ ConvertedType::DECIMAL => {
- // For decimal type we should print precision and scale if they
- // are > 0, e.g. DECIMAL(9, 2) -
- // DECIMAL(9) - DECIMAL
- let precision_scale = match (precision, scale) {
- (p, s) if p > 0 && s > 0 => {
- format!("{}, {}", p, s)
- }
- (p, 0) if p > 0 => format!("{}", p),
- _ => format!(""),
- };
- format!("{}{}", decimal, precision_scale)
- }
- other_converted_type => {
- format!("{}", other_converted_type)
- }
- }
- }
- }
-}
-
-#[allow(unused_must_use)]
-impl<'a> Printer<'a> {
- pub fn print(&mut self, tp: &Type) {
- self.print_indent();
- match *tp {
- Type::PrimitiveType {
- ref basic_info,
- physical_type,
- type_length,
- scale,
- precision,
- } => {
- let phys_type_str = match physical_type {
- PhysicalType::FIXED_LEN_BYTE_ARRAY => {
- // We need to include length for fixed byte array
- format!("{} ({})", physical_type, type_length)
- }
- _ => format!("{}", physical_type),
- };
- // Also print logical type if it is available
- // If there is a logical type, do not print converted type
- let logical_type_str = print_logical_and_converted(
- &basic_info.logical_type(),
- basic_info.converted_type(),
- scale,
- precision,
- );
- if logical_type_str.is_empty() {
- write!(
- self.output,
- "{} {} {};",
- basic_info.repetition(),
- phys_type_str,
- basic_info.name()
- );
- } else {
- write!(
- self.output,
- "{} {} {} ({});",
- basic_info.repetition(),
- phys_type_str,
- basic_info.name(),
- logical_type_str
- );
- }
- }
- Type::GroupType {
- ref basic_info,
- ref fields,
- } => {
- if basic_info.has_repetition() {
- let r = basic_info.repetition();
- write!(self.output, "{} group {} ", r, basic_info.name());
- let logical_str = print_logical_and_converted(
- &basic_info.logical_type(),
- basic_info.converted_type(),
- 0,
- 0,
- );
- if !logical_str.is_empty() {
- write!(self.output, "({}) ", logical_str);
- }
- writeln!(self.output, "{{");
- } else {
- writeln!(self.output, "message {} {{", basic_info.name());
- }
-
- self.indent += INDENT_WIDTH;
- for c in fields {
- self.print(&c);
- writeln!(self.output);
- }
- self.indent -= INDENT_WIDTH;
- self.print_indent();
- write!(self.output, "}}");
- }
- }
- }
-}
-
-#[cfg(test)]
-mod tests {
- use super::*;
-
- use std::sync::Arc;
-
- use crate::basic::{
- DateType, DecimalType, IntType, LogicalType, Repetition, TimeType, TimestampType,
- Type as PhysicalType,
- };
- use crate::errors::Result;
- use crate::schema::{parser::parse_message_type, types::Type};
-
- fn assert_print_parse_message(message: Type) {
- let mut s = String::new();
- {
- let mut p = Printer::new(&mut s);
- p.print(&message);
- }
- println!("{}", &s);
- let parsed = parse_message_type(&s).unwrap();
- assert_eq!(message, parsed);
- }
-
- #[test]
- fn test_print_primitive_type() {
- let mut s = String::new();
- {
- let mut p = Printer::new(&mut s);
- let field = Type::primitive_type_builder("field", PhysicalType::INT32)
- .with_repetition(Repetition::REQUIRED)
- .with_converted_type(ConvertedType::INT_32)
- .build()
- .unwrap();
- p.print(&field);
- }
- assert_eq!(&mut s, "REQUIRED INT32 field (INT_32);");
- }
-
- #[inline]
- fn build_primitive_type(
- name: &str,
- physical_type: PhysicalType,
- logical_type: Option<LogicalType>,
- converted_type: ConvertedType,
- repetition: Repetition,
- ) -> Result<Type> {
- Type::primitive_type_builder(name, physical_type)
- .with_repetition(repetition)
- .with_logical_type(logical_type)
- .with_converted_type(converted_type)
- .build()
- }
-
- #[test]
- fn test_print_logical_types() {
- let types_and_strings = vec![
- (
- build_primitive_type(
- "field",
- PhysicalType::INT32,
- Some(LogicalType::INTEGER(IntType {
- bit_width: 32,
- is_signed: true,
- })),
- ConvertedType::NONE,
- Repetition::REQUIRED,
- )
- .unwrap(),
- "REQUIRED INT32 field (INTEGER(32,true));",
- ),
- (
- build_primitive_type(
- "field",
- PhysicalType::INT32,
- Some(LogicalType::INTEGER(IntType {
- bit_width: 8,
- is_signed: false,
- })),
- ConvertedType::NONE,
- Repetition::OPTIONAL,
- )
- .unwrap(),
- "OPTIONAL INT32 field (INTEGER(8,false));",
- ),
- (
- build_primitive_type(
- "field",
- PhysicalType::INT32,
- Some(LogicalType::INTEGER(IntType {
- bit_width: 16,
- is_signed: true,
- })),
- ConvertedType::INT_16,
- Repetition::REPEATED,
- )
- .unwrap(),
- "REPEATED INT32 field (INTEGER(16,true));",
- ),
- (
- build_primitive_type(
- "field",
- PhysicalType::INT64,
- None,
- ConvertedType::NONE,
- Repetition::REPEATED,
- )
- .unwrap(),
- "REPEATED INT64 field;",
- ),
- (
- build_primitive_type(
- "field",
- PhysicalType::FLOAT,
- None,
- ConvertedType::NONE,
- Repetition::REQUIRED,
- )
- .unwrap(),
- "REQUIRED FLOAT field;",
- ),
- (
- build_primitive_type(
- "booleans",
- PhysicalType::BOOLEAN,
- None,
- ConvertedType::NONE,
- Repetition::OPTIONAL,
- )
- .unwrap(),
- "OPTIONAL BOOLEAN booleans;",
- ),
- (
- build_primitive_type(
- "field",
- PhysicalType::INT64,
- Some(LogicalType::TIMESTAMP(TimestampType {
- is_adjusted_to_u_t_c: true,
- unit: TimeUnit::MILLIS(Default::default()),
- })),
- ConvertedType::NONE,
- Repetition::REQUIRED,
- )
- .unwrap(),
- "REQUIRED INT64 field (TIMESTAMP(MILLIS,true));",
- ),
- (
- build_primitive_type(
- "field",
- PhysicalType::INT32,
- Some(LogicalType::DATE(DateType {})),
- ConvertedType::NONE,
- Repetition::OPTIONAL,
- )
- .unwrap(),
- "OPTIONAL INT32 field (DATE);",
- ),
- (
- build_primitive_type(
- "field",
- PhysicalType::INT32,
- Some(LogicalType::TIME(TimeType {
- unit: TimeUnit::MILLIS(Default::default()),
- is_adjusted_to_u_t_c: false,
- })),
- ConvertedType::TIME_MILLIS,
- Repetition::REQUIRED,
- )
- .unwrap(),
- "REQUIRED INT32 field (TIME(MILLIS,false));",
- ),
- (
- build_primitive_type(
- "field",
- PhysicalType::BYTE_ARRAY,
- None,
- ConvertedType::NONE,
- Repetition::REQUIRED,
- )
- .unwrap(),
- "REQUIRED BYTE_ARRAY field;",
- ),
- (
- build_primitive_type(
- "field",
- PhysicalType::BYTE_ARRAY,
- None,
- ConvertedType::UTF8,
- Repetition::REQUIRED,
- )
- .unwrap(),
- "REQUIRED BYTE_ARRAY field (UTF8);",
- ),
- (
- build_primitive_type(
- "field",
- PhysicalType::BYTE_ARRAY,
- Some(LogicalType::JSON(Default::default())),
- ConvertedType::JSON,
- Repetition::REQUIRED,
- )
- .unwrap(),
- "REQUIRED BYTE_ARRAY field (JSON);",
- ),
- (
- build_primitive_type(
- "field",
- PhysicalType::BYTE_ARRAY,
- Some(LogicalType::BSON(Default::default())),
- ConvertedType::BSON,
- Repetition::REQUIRED,
- )
- .unwrap(),
- "REQUIRED BYTE_ARRAY field (BSON);",
- ),
- (
- build_primitive_type(
- "field",
- PhysicalType::BYTE_ARRAY,
- Some(LogicalType::STRING(Default::default())),
- ConvertedType::NONE,
- Repetition::REQUIRED,
- )
- .unwrap(),
- "REQUIRED BYTE_ARRAY field (STRING);",
- ),
- ];
-
- types_and_strings.into_iter().for_each(|(field, expected)| {
- let mut s = String::new();
- {
- let mut p = Printer::new(&mut s);
- p.print(&field);
- }
- assert_eq!(&s, expected)
- });
- }
-
- #[inline]
- fn decimal_length_from_precision(precision: usize) -> i32 {
- (10.0_f64.powi(precision as i32).log2() / 8.0).ceil() as i32
- }
-
- #[test]
- fn test_print_flba_logical_types() {
- let types_and_strings = vec![
- (
- Type::primitive_type_builder("field", PhysicalType::FIXED_LEN_BYTE_ARRAY)
- .with_logical_type(None)
- .with_converted_type(ConvertedType::INTERVAL)
- .with_length(12)
- .with_repetition(Repetition::REQUIRED)
- .build()
- .unwrap(),
- "REQUIRED FIXED_LEN_BYTE_ARRAY (12) field (INTERVAL);",
- ),
- (
- Type::primitive_type_builder("field", PhysicalType::FIXED_LEN_BYTE_ARRAY)
- .with_logical_type(Some(LogicalType::UUID(Default::default())))
- .with_length(16)
- .with_repetition(Repetition::REQUIRED)
- .build()
- .unwrap(),
- "REQUIRED FIXED_LEN_BYTE_ARRAY (16) field (UUID);",
- ),
- (
- Type::primitive_type_builder(
- "decimal",
- PhysicalType::FIXED_LEN_BYTE_ARRAY,
- )
- .with_logical_type(Some(LogicalType::DECIMAL(DecimalType {
- precision: 32,
- scale: 20,
- })))
- .with_precision(32)
- .with_scale(20)
- .with_length(decimal_length_from_precision(32))
- .with_repetition(Repetition::REPEATED)
- .build()
- .unwrap(),
- "REPEATED FIXED_LEN_BYTE_ARRAY (14) decimal (DECIMAL(32,20));",
- ),
- ];
-
- types_and_strings.into_iter().for_each(|(field, expected)| {
- let mut s = String::new();
- {
- let mut p = Printer::new(&mut s);
- p.print(&field);
- }
- assert_eq!(&s, expected)
- });
- }
-
- #[test]
- fn test_print_group_type() {
- let mut s = String::new();
- {
- let mut p = Printer::new(&mut s);
- let f1 = Type::primitive_type_builder("f1", PhysicalType::INT32)
- .with_repetition(Repetition::REQUIRED)
- .with_converted_type(ConvertedType::INT_32)
- .with_id(0)
- .build();
- let f2 = Type::primitive_type_builder("f2", PhysicalType::BYTE_ARRAY)
- .with_converted_type(ConvertedType::UTF8)
- .with_id(1)
- .build();
- let f3 = Type::primitive_type_builder("f3", PhysicalType::BYTE_ARRAY)
- .with_logical_type(Some(LogicalType::STRING(Default::default())))
- .with_id(1)
- .build();
- let f4 =
- Type::primitive_type_builder("f4", PhysicalType::FIXED_LEN_BYTE_ARRAY)
- .with_repetition(Repetition::REPEATED)
- .with_converted_type(ConvertedType::INTERVAL)
- .with_length(12)
- .with_id(2)
- .build();
- let mut struct_fields = Vec::new();
- struct_fields.push(Arc::new(f1.unwrap()));
- struct_fields.push(Arc::new(f2.unwrap()));
- struct_fields.push(Arc::new(f3.unwrap()));
- let field = Type::group_type_builder("field")
- .with_repetition(Repetition::OPTIONAL)
- .with_fields(&mut struct_fields)
- .with_id(1)
- .build()
- .unwrap();
- let mut fields = Vec::new();
- fields.push(Arc::new(field));
- fields.push(Arc::new(f4.unwrap()));
- let message = Type::group_type_builder("schema")
- .with_fields(&mut fields)
- .with_id(2)
- .build()
- .unwrap();
- p.print(&message);
- }
- let expected = "message schema {
- OPTIONAL group field {
- REQUIRED INT32 f1 (INT_32);
- OPTIONAL BYTE_ARRAY f2 (UTF8);
- OPTIONAL BYTE_ARRAY f3 (STRING);
- }
- REPEATED FIXED_LEN_BYTE_ARRAY (12) f4 (INTERVAL);
-}";
- assert_eq!(&mut s, expected);
- }
-
- #[test]
- fn test_print_and_parse_primitive() {
- let a2 = Type::primitive_type_builder("a2", PhysicalType::BYTE_ARRAY)
- .with_repetition(Repetition::REPEATED)
- .with_converted_type(ConvertedType::UTF8)
- .build()
- .unwrap();
-
- let a1 = Type::group_type_builder("a1")
- .with_repetition(Repetition::OPTIONAL)
- .with_logical_type(Some(LogicalType::LIST(Default::default())))
- .with_converted_type(ConvertedType::LIST)
- .with_fields(&mut vec![Arc::new(a2)])
- .build()
- .unwrap();
-
- let b3 = Type::primitive_type_builder("b3", PhysicalType::INT32)
- .with_repetition(Repetition::OPTIONAL)
- .build()
- .unwrap();
-
- let b4 = Type::primitive_type_builder("b4", PhysicalType::DOUBLE)
- .with_repetition(Repetition::OPTIONAL)
- .build()
- .unwrap();
-
- let b2 = Type::group_type_builder("b2")
- .with_repetition(Repetition::REPEATED)
- .with_converted_type(ConvertedType::NONE)
- .with_fields(&mut vec![Arc::new(b3), Arc::new(b4)])
- .build()
- .unwrap();
-
- let b1 = Type::group_type_builder("b1")
- .with_repetition(Repetition::OPTIONAL)
- .with_logical_type(Some(LogicalType::LIST(Default::default())))
- .with_converted_type(ConvertedType::LIST)
- .with_fields(&mut vec![Arc::new(b2)])
- .build()
- .unwrap();
-
- let a0 = Type::group_type_builder("a0")
- .with_repetition(Repetition::REQUIRED)
- .with_fields(&mut vec![Arc::new(a1), Arc::new(b1)])
- .build()
- .unwrap();
-
- let message = Type::group_type_builder("root")
- .with_fields(&mut vec![Arc::new(a0)])
- .build()
- .unwrap();
-
- assert_print_parse_message(message);
- }
-
- #[test]
- fn test_print_and_parse_nested() {
- let f1 = Type::primitive_type_builder("f1", PhysicalType::INT32)
- .with_repetition(Repetition::REQUIRED)
- .with_converted_type(ConvertedType::INT_32)
- .build()
- .unwrap();
-
- let f2 = Type::primitive_type_builder("f2", PhysicalType::BYTE_ARRAY)
- .with_repetition(Repetition::OPTIONAL)
- .with_converted_type(ConvertedType::UTF8)
- .build()
- .unwrap();
-
- let field = Type::group_type_builder("field")
- .with_repetition(Repetition::OPTIONAL)
- .with_fields(&mut vec![Arc::new(f1), Arc::new(f2)])
- .build()
- .unwrap();
-
- let f3 = Type::primitive_type_builder("f3", PhysicalType::FIXED_LEN_BYTE_ARRAY)
- .with_repetition(Repetition::REPEATED)
- .with_converted_type(ConvertedType::INTERVAL)
- .with_length(12)
- .build()
- .unwrap();
-
- let message = Type::group_type_builder("schema")
- .with_fields(&mut vec![Arc::new(field), Arc::new(f3)])
- .build()
- .unwrap();
-
- assert_print_parse_message(message);
- }
-
- #[test]
- fn test_print_and_parse_decimal() {
- let f1 = Type::primitive_type_builder("f1", PhysicalType::INT32)
- .with_repetition(Repetition::OPTIONAL)
- .with_logical_type(Some(LogicalType::DECIMAL(DecimalType {
- precision: 9,
- scale: 2,
- })))
- .with_converted_type(ConvertedType::DECIMAL)
- .with_precision(9)
- .with_scale(2)
- .build()
- .unwrap();
-
- let f2 = Type::primitive_type_builder("f2", PhysicalType::INT32)
- .with_repetition(Repetition::OPTIONAL)
- .with_logical_type(Some(LogicalType::DECIMAL(DecimalType {
- precision: 9,
- scale: 0,
- })))
- .with_converted_type(ConvertedType::DECIMAL)
- .with_precision(9)
- .with_scale(0)
- .build()
- .unwrap();
-
- let message = Type::group_type_builder("schema")
- .with_fields(&mut vec![Arc::new(f1), Arc::new(f2)])
- .build()
- .unwrap();
-
- assert_print_parse_message(message);
- }
-}
diff --git a/parquet/src/schema/types.rs b/parquet/src/schema/types.rs
deleted file mode 100644
index 1aa8c26..0000000
--- a/parquet/src/schema/types.rs
+++ /dev/null
@@ -1,2080 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-//! Contains structs and methods to build Parquet schema and schema descriptors.
-
-use std::{collections::HashMap, convert::From, fmt, sync::Arc};
-
-use parquet_format::SchemaElement;
-
-use crate::basic::{
- ConvertedType, LogicalType, Repetition, TimeType, TimeUnit, Type as PhysicalType,
-};
-use crate::errors::{ParquetError, Result};
-
-// ----------------------------------------------------------------------
-// Parquet Type definitions
-
-/// Type alias for `Arc<Type>`.
-pub type TypePtr = Arc<Type>;
-/// Type alias for `Arc<SchemaDescriptor>`.
-pub type SchemaDescPtr = Arc<SchemaDescriptor>;
-/// Type alias for `Arc<ColumnDescriptor>`.
-pub type ColumnDescPtr = Arc<ColumnDescriptor>;
-
-/// Representation of a Parquet type.
-/// Used to describe primitive leaf fields and structs, including top-level schema.
-/// Note that the top-level schema type is represented using `GroupType` whose
-/// repetition is `None`.
-#[derive(Clone, Debug, PartialEq)]
-pub enum Type {
- PrimitiveType {
- basic_info: BasicTypeInfo,
- physical_type: PhysicalType,
- type_length: i32,
- scale: i32,
- precision: i32,
- },
- GroupType {
- basic_info: BasicTypeInfo,
- fields: Vec<TypePtr>,
- },
-}
-
-impl Type {
- /// Creates primitive type builder with provided field name and physical type.
- pub fn primitive_type_builder(
- name: &str,
- physical_type: PhysicalType,
- ) -> PrimitiveTypeBuilder {
- PrimitiveTypeBuilder::new(name, physical_type)
- }
-
- /// Creates group type builder with provided column name.
- pub fn group_type_builder(name: &str) -> GroupTypeBuilder {
- GroupTypeBuilder::new(name)
- }
-
- /// Returns [`BasicTypeInfo`] information about the type.
- pub fn get_basic_info(&self) -> &BasicTypeInfo {
- match *self {
- Type::PrimitiveType { ref basic_info, .. } => &basic_info,
- Type::GroupType { ref basic_info, .. } => &basic_info,
- }
- }
-
- /// Returns this type's field name.
- pub fn name(&self) -> &str {
- self.get_basic_info().name()
- }
-
- /// Gets the fields from this group type.
- /// Note that this will panic if called on a non-group type.
- // TODO: should we return `&[&Type]` here?
- pub fn get_fields(&self) -> &[TypePtr] {
- match *self {
- Type::GroupType { ref fields, .. } => &fields[..],
- _ => panic!("Cannot call get_fields() on a non-group type"),
- }
- }
-
- /// Gets physical type of this primitive type.
- /// Note that this will panic if called on a non-primitive type.
- pub fn get_physical_type(&self) -> PhysicalType {
- match *self {
- Type::PrimitiveType {
- basic_info: _,
- physical_type,
- ..
- } => physical_type,
- _ => panic!("Cannot call get_physical_type() on a non-primitive type"),
- }
- }
-
- /// Gets precision of this primitive type.
- /// Note that this will panic if called on a non-primitive type.
- pub fn get_precision(&self) -> i32 {
- match *self {
- Type::PrimitiveType { precision, .. } => precision,
- _ => panic!("Cannot call get_precision() on non-primitive type"),
- }
- }
-
- /// Gets scale of this primitive type.
- /// Note that this will panic if called on a non-primitive type.
- pub fn get_scale(&self) -> i32 {
- match *self {
- Type::PrimitiveType { scale, .. } => scale,
- _ => panic!("Cannot call get_scale() on non-primitive type"),
- }
- }
-
- /// Checks if `sub_type` schema is part of current schema.
- /// This method can be used to check if projected columns are part of the root schema.
- pub fn check_contains(&self, sub_type: &Type) -> bool {
- // Names match, and repetitions match or not set for both
- let basic_match = self.get_basic_info().name()
- == sub_type.get_basic_info().name()
- && (self.is_schema() && sub_type.is_schema()
- || !self.is_schema()
- && !sub_type.is_schema()
- && self.get_basic_info().repetition()
- == sub_type.get_basic_info().repetition());
-
- match *self {
- Type::PrimitiveType { .. } if basic_match && sub_type.is_primitive() => {
- self.get_physical_type() == sub_type.get_physical_type()
- }
- Type::GroupType { .. } if basic_match && sub_type.is_group() => {
- // build hashmap of name -> TypePtr
- let mut field_map = HashMap::new();
- for field in self.get_fields() {
- field_map.insert(field.name(), field);
- }
-
- for field in sub_type.get_fields() {
- if !field_map
- .get(field.name())
- .map(|tpe| tpe.check_contains(field))
- .unwrap_or(false)
- {
- return false;
- }
- }
- true
- }
- _ => false,
- }
- }
-
- /// Returns `true` if this type is a primitive type, `false` otherwise.
- pub fn is_primitive(&self) -> bool {
- matches!(*self, Type::PrimitiveType { .. })
- }
-
- /// Returns `true` if this type is a group type, `false` otherwise.
- pub fn is_group(&self) -> bool {
- matches!(*self, Type::GroupType { .. })
- }
-
- /// Returns `true` if this type is the top-level schema type (message type).
- pub fn is_schema(&self) -> bool {
- match *self {
- Type::GroupType { ref basic_info, .. } => !basic_info.has_repetition(),
- _ => false,
- }
- }
-
- /// Returns `true` if this type is repeated or optional.
- /// If this type doesn't have repetition defined, we still treat it as optional.
- pub fn is_optional(&self) -> bool {
- self.get_basic_info().has_repetition()
- && self.get_basic_info().repetition() != Repetition::REQUIRED
- }
-}
-
-/// A builder for primitive types. All attributes are optional
-/// except the name and physical type.
-/// Note that if not specified explicitly, `Repetition::OPTIONAL` is used.
-pub struct PrimitiveTypeBuilder<'a> {
- name: &'a str,
- repetition: Repetition,
- physical_type: PhysicalType,
- converted_type: ConvertedType,
- logical_type: Option<LogicalType>,
- length: i32,
- precision: i32,
- scale: i32,
- id: Option<i32>,
-}
-
-impl<'a> PrimitiveTypeBuilder<'a> {
- /// Creates new primitive type builder with provided field name and physical type.
- pub fn new(name: &'a str, physical_type: PhysicalType) -> Self {
- Self {
- name,
- repetition: Repetition::OPTIONAL,
- physical_type,
- converted_type: ConvertedType::NONE,
- logical_type: None,
- length: -1,
- precision: -1,
- scale: -1,
- id: None,
- }
- }
-
- /// Sets [`Repetition`](crate::basic::Repetition) for this field and returns itself.
- pub fn with_repetition(mut self, repetition: Repetition) -> Self {
- self.repetition = repetition;
- self
- }
-
- /// Sets [`ConvertedType`](crate::basic::ConvertedType) for this field and returns itself.
- pub fn with_converted_type(mut self, converted_type: ConvertedType) -> Self {
- self.converted_type = converted_type;
- self
- }
-
- /// Sets [`LogicalType`](crate::basic::LogicalType) for this field and returns itself.
- /// If only the logical type is populated for a primitive type, the converted type
- /// will be automatically populated, and can thus be omitted.
- pub fn with_logical_type(mut self, logical_type: Option<LogicalType>) -> Self {
- self.logical_type = logical_type;
- self
- }
-
- /// Sets type length and returns itself.
- /// This is only applied to FIXED_LEN_BYTE_ARRAY and INT96 (INTERVAL) types, because
- /// they maintain fixed size underlying byte array.
- /// By default, value is `0`.
- pub fn with_length(mut self, length: i32) -> Self {
- self.length = length;
- self
- }
-
- /// Sets precision for Parquet DECIMAL physical type and returns itself.
- /// By default, it equals to `0` and used only for decimal context.
- pub fn with_precision(mut self, precision: i32) -> Self {
- self.precision = precision;
- self
- }
-
- /// Sets scale for Parquet DECIMAL physical type and returns itself.
- /// By default, it equals to `0` and used only for decimal context.
- pub fn with_scale(mut self, scale: i32) -> Self {
- self.scale = scale;
- self
- }
-
- /// Sets optional field id and returns itself.
- pub fn with_id(mut self, id: i32) -> Self {
- self.id = Some(id);
- self
- }
-
- /// Creates a new `PrimitiveType` instance from the collected attributes.
- /// Returns `Err` in case of any building conditions are not met.
- pub fn build(self) -> Result<Type> {
- let mut basic_info = BasicTypeInfo {
- name: String::from(self.name),
- repetition: Some(self.repetition),
- converted_type: self.converted_type,
- logical_type: self.logical_type.clone(),
- id: self.id,
- };
-
- // Check length before logical type, since it is used for logical type validation.
- if self.physical_type == PhysicalType::FIXED_LEN_BYTE_ARRAY && self.length < 0 {
- return Err(general_err!(
- "Invalid FIXED_LEN_BYTE_ARRAY length: {}",
- self.length
- ));
- }
-
- match &self.logical_type {
- Some(logical_type) => {
- // If a converted type is populated, check that it is consistent with
- // its logical type
- if self.converted_type != ConvertedType::NONE {
- if ConvertedType::from(self.logical_type.clone())
- != self.converted_type
- {
- return Err(general_err!(
- "Logical type {:?} is imcompatible with converted type {}",
- logical_type,
- self.converted_type
- ));
- }
- } else {
- // Populate the converted type for backwards compatibility
- basic_info.converted_type = self.logical_type.clone().into();
- }
- // Check that logical type and physical type are compatible
- match (logical_type, self.physical_type) {
- (LogicalType::MAP(_), _) | (LogicalType::LIST(_), _) => {
- return Err(general_err!(
- "{:?} cannot be applied to a primitive type",
- logical_type
- ));
- }
- (LogicalType::ENUM(_), PhysicalType::BYTE_ARRAY) => {}
- (LogicalType::DECIMAL(t), _) => {
- // Check that scale and precision are consistent with legacy values
- if t.scale != self.scale {
- return Err(general_err!(
- "DECIMAL logical type scale {} must match self.scale {}",
- t.scale,
- self.scale
- ));
- }
- if t.precision != self.precision {
- return Err(general_err!(
- "DECIMAL logical type precision {} must match self.precision {}",
- t.precision,
- self.precision
- ));
- }
- self.check_decimal_precision_scale()?;
- }
- (LogicalType::DATE(_), PhysicalType::INT32) => {}
- (
- LogicalType::TIME(TimeType {
- unit: TimeUnit::MILLIS(_),
- ..
- }),
- PhysicalType::INT32,
- ) => {}
- (LogicalType::TIME(t), PhysicalType::INT64) => {
- if t.unit == TimeUnit::MILLIS(Default::default()) {
- return Err(general_err!(
- "Cannot use millisecond unit on INT64 type"
- ));
- }
- }
- (LogicalType::TIMESTAMP(_), PhysicalType::INT64) => {}
- (LogicalType::INTEGER(t), PhysicalType::INT32)
- if t.bit_width <= 32 => {}
- (LogicalType::INTEGER(t), PhysicalType::INT64)
- if t.bit_width == 64 => {}
- // Null type
- (LogicalType::UNKNOWN(_), PhysicalType::INT32) => {}
- (LogicalType::STRING(_), PhysicalType::BYTE_ARRAY) => {}
- (LogicalType::JSON(_), PhysicalType::BYTE_ARRAY) => {}
- (LogicalType::BSON(_), PhysicalType::BYTE_ARRAY) => {}
- (LogicalType::UUID(_), PhysicalType::FIXED_LEN_BYTE_ARRAY) => {}
- (a, b) => {
- return Err(general_err!(
- "Cannot annotate {:?} from {} fields",
- a,
- b
- ))
- }
- }
- }
- None => {}
- }
-
- match self.converted_type {
- ConvertedType::NONE => {}
- ConvertedType::UTF8 | ConvertedType::BSON | ConvertedType::JSON => {
- if self.physical_type != PhysicalType::BYTE_ARRAY {
- return Err(general_err!(
- "{} can only annotate BYTE_ARRAY fields",
- self.converted_type
- ));
- }
- }
- ConvertedType::DECIMAL => {
- self.check_decimal_precision_scale()?;
- }
- ConvertedType::DATE
- | ConvertedType::TIME_MILLIS
- | ConvertedType::UINT_8
- | ConvertedType::UINT_16
- | ConvertedType::UINT_32
- | ConvertedType::INT_8
- | ConvertedType::INT_16
- | ConvertedType::INT_32 => {
- if self.physical_type != PhysicalType::INT32 {
- return Err(general_err!(
- "{} can only annotate INT32",
- self.converted_type
- ));
- }
- }
- ConvertedType::TIME_MICROS
- | ConvertedType::TIMESTAMP_MILLIS
- | ConvertedType::TIMESTAMP_MICROS
- | ConvertedType::UINT_64
- | ConvertedType::INT_64 => {
- if self.physical_type != PhysicalType::INT64 {
- return Err(general_err!(
- "{} can only annotate INT64",
- self.converted_type
- ));
- }
- }
- ConvertedType::INTERVAL => {
- if self.physical_type != PhysicalType::FIXED_LEN_BYTE_ARRAY
- || self.length != 12
- {
- return Err(general_err!(
- "INTERVAL can only annotate FIXED_LEN_BYTE_ARRAY(12)"
- ));
- }
- }
- ConvertedType::ENUM => {
- if self.physical_type != PhysicalType::BYTE_ARRAY {
- return Err(general_err!("ENUM can only annotate BYTE_ARRAY fields"));
- }
- }
- _ => {
- return Err(general_err!(
- "{} cannot be applied to a primitive type",
- self.converted_type
- ));
- }
- }
-
- Ok(Type::PrimitiveType {
- basic_info,
- physical_type: self.physical_type,
- type_length: self.length,
- scale: self.scale,
- precision: self.precision,
- })
- }
-
- #[inline]
- fn check_decimal_precision_scale(&self) -> Result<()> {
- match self.physical_type {
- PhysicalType::INT32
- | PhysicalType::INT64
- | PhysicalType::BYTE_ARRAY
- | PhysicalType::FIXED_LEN_BYTE_ARRAY => (),
- _ => {
- return Err(general_err!(
- "DECIMAL can only annotate INT32, INT64, BYTE_ARRAY and FIXED_LEN_BYTE_ARRAY"
- ));
- }
- }
-
- // Precision is required and must be a non-zero positive integer.
- if self.precision < 1 {
- return Err(general_err!(
- "Invalid DECIMAL precision: {}",
- self.precision
- ));
- }
-
- // Scale must be zero or a positive integer less than the precision.
- if self.scale < 0 {
- return Err(general_err!("Invalid DECIMAL scale: {}", self.scale));
- }
-
- if self.scale >= self.precision {
- return Err(general_err!(
- "Invalid DECIMAL: scale ({}) cannot be greater than or equal to precision \
- ({})",
- self.scale,
- self.precision
- ));
- }
-
- // Check precision and scale based on physical type limitations.
- match self.physical_type {
- PhysicalType::INT32 => {
- if self.precision > 9 {
- return Err(general_err!(
- "Cannot represent INT32 as DECIMAL with precision {}",
- self.precision
- ));
- }
- }
- PhysicalType::INT64 => {
- if self.precision > 18 {
- return Err(general_err!(
- "Cannot represent INT64 as DECIMAL with precision {}",
- self.precision
- ));
- }
- }
- PhysicalType::FIXED_LEN_BYTE_ARRAY => {
- let max_precision =
- (2f64.powi(8 * self.length - 1) - 1f64).log10().floor() as i32;
-
- if self.precision > max_precision {
- return Err(general_err!(
- "Cannot represent FIXED_LEN_BYTE_ARRAY as DECIMAL with length {} and \
- precision {}. The max precision can only be {}",
- self.length,
- self.precision,
- max_precision
- ));
- }
- }
- _ => (), // For BYTE_ARRAY precision is not limited
- }
-
- Ok(())
- }
-}
-
-/// A builder for group types. All attributes are optional except the name.
-/// Note that if not specified explicitly, `None` is used as the repetition of the group,
-/// which means it is a root (message) type.
-pub struct GroupTypeBuilder<'a> {
- name: &'a str,
- repetition: Option<Repetition>,
- converted_type: ConvertedType,
- logical_type: Option<LogicalType>,
- fields: Vec<TypePtr>,
- id: Option<i32>,
-}
-
-impl<'a> GroupTypeBuilder<'a> {
- /// Creates new group type builder with provided field name.
- pub fn new(name: &'a str) -> Self {
- Self {
- name,
- repetition: None,
- converted_type: ConvertedType::NONE,
- logical_type: None,
- fields: Vec::new(),
- id: None,
- }
- }
-
- /// Sets [`Repetition`](crate::basic::Repetition) for this field and returns itself.
- pub fn with_repetition(mut self, repetition: Repetition) -> Self {
- self.repetition = Some(repetition);
- self
- }
-
- /// Sets [`ConvertedType`](crate::basic::ConvertedType) for this field and returns itself.
- pub fn with_converted_type(mut self, converted_type: ConvertedType) -> Self {
- self.converted_type = converted_type;
- self
- }
-
- /// Sets [`LogicalType`](crate::basic::LogicalType) for this field and returns itself.
- pub fn with_logical_type(mut self, logical_type: Option<LogicalType>) -> Self {
- self.logical_type = logical_type;
- self
- }
-
- /// Sets a list of fields that should be child nodes of this field.
- /// Returns updated self.
- pub fn with_fields(mut self, fields: &mut Vec<TypePtr>) -> Self {
- self.fields.append(fields);
- self
- }
-
- /// Sets optional field id and returns itself.
- pub fn with_id(mut self, id: i32) -> Self {
- self.id = Some(id);
- self
- }
-
- /// Creates a new `GroupType` instance from the gathered attributes.
- pub fn build(self) -> Result<Type> {
- let mut basic_info = BasicTypeInfo {
- name: String::from(self.name),
- repetition: self.repetition,
- converted_type: self.converted_type,
- logical_type: self.logical_type.clone(),
- id: self.id,
- };
- // Populate the converted type if only the logical type is populated
- if self.logical_type.is_some() && self.converted_type == ConvertedType::NONE {
- basic_info.converted_type = self.logical_type.into();
- }
- Ok(Type::GroupType {
- basic_info,
- fields: self.fields,
- })
- }
-}
-
-/// Basic type info. This contains information such as the name of the type,
-/// the repetition level, the logical type and the kind of the type (group, primitive).
-#[derive(Clone, Debug, PartialEq)]
-pub struct BasicTypeInfo {
- name: String,
- repetition: Option<Repetition>,
- converted_type: ConvertedType,
- logical_type: Option<LogicalType>,
- id: Option<i32>,
-}
-
-impl BasicTypeInfo {
- /// Returns field name.
- pub fn name(&self) -> &str {
- &self.name
- }
-
- /// Returns `true` if type has repetition field set, `false` otherwise.
- /// This is mostly applied to group type, because primitive type always has
- /// repetition set.
- pub fn has_repetition(&self) -> bool {
- self.repetition.is_some()
- }
-
- /// Returns [`Repetition`](crate::basic::Repetition) value for the type.
- pub fn repetition(&self) -> Repetition {
- assert!(self.repetition.is_some());
- self.repetition.unwrap()
- }
-
- /// Returns [`ConvertedType`](crate::basic::ConvertedType) value for the type.
- pub fn converted_type(&self) -> ConvertedType {
- self.converted_type
- }
-
- /// Returns [`LogicalType`](crate::basic::LogicalType) value for the type.
- pub fn logical_type(&self) -> Option<LogicalType> {
- // Unlike ConvertedType, LogicalType cannot implement Copy, thus we clone it
- self.logical_type.clone()
- }
-
- /// Returns `true` if id is set, `false` otherwise.
- pub fn has_id(&self) -> bool {
- self.id.is_some()
- }
-
- /// Returns id value for the type.
- pub fn id(&self) -> i32 {
- assert!(self.id.is_some());
- self.id.unwrap()
- }
-}
-
-// ----------------------------------------------------------------------
-// Parquet descriptor definitions
-
-/// Represents a path in a nested schema
-#[derive(Clone, PartialEq, Debug, Eq, Hash)]
-pub struct ColumnPath {
- parts: Vec<String>,
-}
-
-impl ColumnPath {
- /// Creates new column path from vector of field names.
- pub fn new(parts: Vec<String>) -> Self {
- ColumnPath { parts }
- }
-
- /// Returns string representation of this column path.
- /// ```rust
- /// use parquet::schema::types::ColumnPath;
- ///
- /// let path = ColumnPath::new(vec!["a".to_string(), "b".to_string(), "c".to_string()]);
- /// assert_eq!(&path.string(), "a.b.c");
- /// ```
- pub fn string(&self) -> String {
- self.parts.join(".")
- }
-
- /// Appends more components to end of column path.
- /// ```rust
- /// use parquet::schema::types::ColumnPath;
- ///
- /// let mut path = ColumnPath::new(vec!["a".to_string(), "b".to_string(), "c"
- /// .to_string()]);
- /// assert_eq!(&path.string(), "a.b.c");
- ///
- /// path.append(vec!["d".to_string(), "e".to_string()]);
- /// assert_eq!(&path.string(), "a.b.c.d.e");
- /// ```
- pub fn append(&mut self, mut tail: Vec<String>) {
- self.parts.append(&mut tail);
- }
-
- pub fn parts(&self) -> &[String] {
- &self.parts
- }
-}
-
-impl fmt::Display for ColumnPath {
- fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
- write!(f, "{:?}", self.string())
- }
-}
-
-impl From<Vec<String>> for ColumnPath {
- fn from(parts: Vec<String>) -> Self {
- ColumnPath { parts }
- }
-}
-
-impl<'a> From<&'a str> for ColumnPath {
- fn from(single_path: &str) -> Self {
- let s = String::from(single_path);
- ColumnPath::from(s)
- }
-}
-
-impl From<String> for ColumnPath {
- fn from(single_path: String) -> Self {
- let v = vec![single_path];
- ColumnPath { parts: v }
- }
-}
-
-impl AsRef<[String]> for ColumnPath {
- fn as_ref(&self) -> &[String] {
- &self.parts
- }
-}
-
-/// A descriptor for leaf-level primitive columns.
-/// This encapsulates information such as definition and repetition levels and is used to
-/// re-assemble nested data.
-#[derive(Debug, PartialEq)]
-pub struct ColumnDescriptor {
- // The "leaf" primitive type of this column
- primitive_type: TypePtr,
-
- // The maximum definition level for this column
- max_def_level: i16,
-
- // The maximum repetition level for this column
- max_rep_level: i16,
-
- // The path of this column. For instance, "a.b.c.d".
- path: ColumnPath,
-}
-
-impl ColumnDescriptor {
- /// Creates new descriptor for leaf-level column.
- pub fn new(
- primitive_type: TypePtr,
- max_def_level: i16,
- max_rep_level: i16,
- path: ColumnPath,
- ) -> Self {
- Self {
- primitive_type,
- max_def_level,
- max_rep_level,
- path,
- }
- }
-
- /// Returns maximum definition level for this column.
- #[inline]
- pub fn max_def_level(&self) -> i16 {
- self.max_def_level
- }
-
- /// Returns maximum repetition level for this column.
- #[inline]
- pub fn max_rep_level(&self) -> i16 {
- self.max_rep_level
- }
-
- /// Returns [`ColumnPath`] for this column.
- pub fn path(&self) -> &ColumnPath {
- &self.path
- }
-
- /// Returns self type [`Type`](crate::schema::types::Type) for this leaf column.
- pub fn self_type(&self) -> &Type {
- self.primitive_type.as_ref()
- }
-
- /// Returns self type [`TypePtr`](crate::schema::types::TypePtr) for this leaf
- /// column.
- pub fn self_type_ptr(&self) -> TypePtr {
- self.primitive_type.clone()
- }
-
- /// Returns column name.
- pub fn name(&self) -> &str {
- self.primitive_type.name()
- }
-
- /// Returns [`ConvertedType`](crate::basic::ConvertedType) for this column.
- pub fn converted_type(&self) -> ConvertedType {
- self.primitive_type.get_basic_info().converted_type()
- }
-
- /// Returns [`LogicalType`](crate::basic::LogicalType) for this column.
- pub fn logical_type(&self) -> Option<LogicalType> {
- self.primitive_type.get_basic_info().logical_type()
- }
-
- /// Returns physical type for this column.
- /// Note that it will panic if called on a non-primitive type.
- pub fn physical_type(&self) -> PhysicalType {
- match self.primitive_type.as_ref() {
- Type::PrimitiveType { physical_type, .. } => *physical_type,
- _ => panic!("Expected primitive type!"),
- }
- }
-
- /// Returns type length for this column.
- /// Note that it will panic if called on a non-primitive type.
- pub fn type_length(&self) -> i32 {
- match self.primitive_type.as_ref() {
- Type::PrimitiveType { type_length, .. } => *type_length,
- _ => panic!("Expected primitive type!"),
- }
- }
-
- /// Returns type precision for this column.
- /// Note that it will panic if called on a non-primitive type.
- pub fn type_precision(&self) -> i32 {
- match self.primitive_type.as_ref() {
- Type::PrimitiveType { precision, .. } => *precision,
- _ => panic!("Expected primitive type!"),
- }
- }
-
- /// Returns type scale for this column.
- /// Note that it will panic if called on a non-primitive type.
- pub fn type_scale(&self) -> i32 {
- match self.primitive_type.as_ref() {
- Type::PrimitiveType { scale, .. } => *scale,
- _ => panic!("Expected primitive type!"),
- }
- }
-}
-
-/// A schema descriptor. This encapsulates the top-level schemas for all the columns,
-/// as well as all descriptors for all the primitive columns.
-pub struct SchemaDescriptor {
- // The top-level schema (the "message" type).
- // This must be a `GroupType` where each field is a root column type in the schema.
- schema: TypePtr,
-
- // All the descriptors for primitive columns in this schema, constructed from
- // `schema` in DFS order.
- leaves: Vec<ColumnDescPtr>,
-
- // Mapping from a leaf column's index to the root column type that it
- // comes from. For instance: the leaf `a.b.c.d` would have a link back to `a`:
- // -- a <-----+
- // -- -- b |
- // -- -- -- c |
- // -- -- -- -- d
- leaf_to_base: Vec<TypePtr>,
-}
-
-impl fmt::Debug for SchemaDescriptor {
- fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
- // Skip leaves and leaf_to_base as they only a cache information already found in `schema`
- f.debug_struct("SchemaDescriptor")
- .field("schema", &self.schema)
- .finish()
- }
-}
-
-impl SchemaDescriptor {
- /// Creates new schema descriptor from Parquet schema.
- pub fn new(tp: TypePtr) -> Self {
- assert!(tp.is_group(), "SchemaDescriptor should take a GroupType");
- let mut leaves = vec![];
- let mut leaf_to_base = Vec::new();
- for f in tp.get_fields() {
- let mut path = vec![];
- build_tree(f, f, 0, 0, &mut leaves, &mut leaf_to_base, &mut path);
- }
-
- Self {
- schema: tp,
- leaves,
- leaf_to_base,
- }
- }
-
- /// Returns [`ColumnDescriptor`] for a field position.
- pub fn column(&self, i: usize) -> ColumnDescPtr {
- assert!(
- i < self.leaves.len(),
- "Index out of bound: {} not in [0, {})",
- i,
- self.leaves.len()
- );
- self.leaves[i].clone()
- }
-
- /// Returns slice of [`ColumnDescriptor`].
- pub fn columns(&self) -> &[ColumnDescPtr] {
- &self.leaves
- }
-
- /// Returns number of leaf-level columns.
- pub fn num_columns(&self) -> usize {
- self.leaves.len()
- }
-
- /// Returns column root [`Type`](crate::schema::types::Type) for a field position.
- pub fn get_column_root(&self, i: usize) -> &Type {
- let result = self.column_root_of(i);
- result.as_ref()
- }
-
- /// Returns column root [`Type`](crate::schema::types::Type) pointer for a field
- /// position.
- pub fn get_column_root_ptr(&self, i: usize) -> TypePtr {
- let result = self.column_root_of(i);
- result.clone()
- }
-
- fn column_root_of(&self, i: usize) -> &Arc<Type> {
- assert!(
- i < self.leaves.len(),
- "Index out of bound: {} not in [0, {})",
- i,
- self.leaves.len()
- );
-
- self.leaf_to_base
- .get(i)
- .unwrap_or_else(|| panic!("Expected a value for index {} but found None", i))
- }
-
- /// Returns schema as [`Type`](crate::schema::types::Type).
- pub fn root_schema(&self) -> &Type {
- self.schema.as_ref()
- }
-
- pub fn root_schema_ptr(&self) -> TypePtr {
- self.schema.clone()
- }
-
- /// Returns schema name.
- pub fn name(&self) -> &str {
- self.schema.name()
- }
-}
-
-fn build_tree<'a>(
- tp: &'a TypePtr,
- base_tp: &TypePtr,
- mut max_rep_level: i16,
- mut max_def_level: i16,
- leaves: &mut Vec<ColumnDescPtr>,
- leaf_to_base: &mut Vec<TypePtr>,
- path_so_far: &mut Vec<&'a str>,
-) {
- assert!(tp.get_basic_info().has_repetition());
-
- path_so_far.push(tp.name());
- match tp.get_basic_info().repetition() {
- Repetition::OPTIONAL => {
- max_def_level += 1;
- }
- Repetition::REPEATED => {
- max_def_level += 1;
- max_rep_level += 1;
- }
- _ => {}
- }
-
- match tp.as_ref() {
- Type::PrimitiveType { .. } => {
- let mut path: Vec<String> = vec![];
- path.extend(path_so_far.iter().copied().map(String::from));
- leaves.push(Arc::new(ColumnDescriptor::new(
- tp.clone(),
- max_def_level,
- max_rep_level,
- ColumnPath::new(path),
- )));
- leaf_to_base.push(base_tp.clone());
- }
- Type::GroupType { ref fields, .. } => {
- for f in fields {
- build_tree(
- f,
- base_tp,
- max_rep_level,
- max_def_level,
- leaves,
- leaf_to_base,
- path_so_far,
- );
- path_so_far.pop();
- }
- }
- }
-}
-
-/// Method to convert from Thrift.
-pub fn from_thrift(elements: &[SchemaElement]) -> Result<TypePtr> {
- let mut index = 0;
- let mut schema_nodes = Vec::new();
- while index < elements.len() {
- let t = from_thrift_helper(elements, index)?;
- index = t.0;
- schema_nodes.push(t.1);
- }
- if schema_nodes.len() != 1 {
- return Err(general_err!(
- "Expected exactly one root node, but found {}",
- schema_nodes.len()
- ));
- }
-
- Ok(schema_nodes.remove(0))
-}
-
-/// Constructs a new Type from the `elements`, starting at index `index`.
-/// The first result is the starting index for the next Type after this one. If it is
-/// equal to `elements.len()`, then this Type is the last one.
-/// The second result is the result Type.
-fn from_thrift_helper(
- elements: &[SchemaElement],
- index: usize,
-) -> Result<(usize, TypePtr)> {
- // Whether or not the current node is root (message type).
- // There is only one message type node in the schema tree.
- let is_root_node = index == 0;
-
- if index > elements.len() {
- return Err(general_err!(
- "Index out of bound, index = {}, len = {}",
- index,
- elements.len()
- ));
- }
- let element = &elements[index];
- let converted_type = ConvertedType::from(element.converted_type);
- // LogicalType is only present in v2 Parquet files. ConvertedType is always
- // populated, regardless of the version of the file (v1 or v2).
- let logical_type = element
- .logical_type
- .as_ref()
- .map(|value| LogicalType::from(value.clone()));
- let field_id = elements[index].field_id;
- match elements[index].num_children {
- // From parquet-format:
- // The children count is used to construct the nested relationship.
- // This field is not set when the element is a primitive type
- // Sometimes parquet-cpp sets num_children field to 0 for primitive types, so we
- // have to handle this case too.
- None | Some(0) => {
- // primitive type
- if elements[index].repetition_type.is_none() {
- return Err(general_err!(
- "Repetition level must be defined for a primitive type"
- ));
- }
- let repetition = Repetition::from(elements[index].repetition_type.unwrap());
- let physical_type = PhysicalType::from(elements[index].type_.unwrap());
- let length = elements[index].type_length.unwrap_or(-1);
- let scale = elements[index].scale.unwrap_or(-1);
- let precision = elements[index].precision.unwrap_or(-1);
- let name = &elements[index].name;
- let mut builder = Type::primitive_type_builder(name, physical_type)
- .with_repetition(repetition)
- .with_converted_type(converted_type)
- .with_logical_type(logical_type)
- .with_length(length)
- .with_precision(precision)
- .with_scale(scale);
- if let Some(id) = field_id {
- builder = builder.with_id(id);
- }
- Ok((index + 1, Arc::new(builder.build()?)))
- }
- Some(n) => {
- let repetition = elements[index].repetition_type.map(Repetition::from);
- let mut fields = vec![];
- let mut next_index = index + 1;
- for _ in 0..n {
- let child_result = from_thrift_helper(elements, next_index as usize)?;
- next_index = child_result.0;
- fields.push(child_result.1);
- }
-
- let mut builder = Type::group_type_builder(&elements[index].name)
- .with_converted_type(converted_type)
- .with_logical_type(logical_type)
- .with_fields(&mut fields);
- if let Some(rep) = repetition {
- // Sometimes parquet-cpp and parquet-mr set repetition level REQUIRED or
- // REPEATED for root node.
- //
- // We only set repetition for group types that are not top-level message
- // type. According to parquet-format:
- // Root of the schema does not have a repetition_type.
- // All other types must have one.
- if !is_root_node {
- builder = builder.with_repetition(rep);
- }
- }
- if let Some(id) = field_id {
- builder = builder.with_id(id);
- }
- Ok((next_index, Arc::new(builder.build().unwrap())))
- }
- }
-}
-
-/// Method to convert to Thrift.
-pub fn to_thrift(schema: &Type) -> Result<Vec<SchemaElement>> {
- if !schema.is_group() {
- return Err(general_err!("Root schema must be Group type"));
- }
- let mut elements: Vec<SchemaElement> = Vec::new();
- to_thrift_helper(schema, &mut elements);
- Ok(elements)
-}
-
-/// Constructs list of `SchemaElement` from the schema using depth-first traversal.
-/// Here we assume that schema is always valid and starts with group type.
-fn to_thrift_helper(schema: &Type, elements: &mut Vec<SchemaElement>) {
- match *schema {
- Type::PrimitiveType {
- ref basic_info,
- physical_type,
- type_length,
- scale,
- precision,
- } => {
- let element = SchemaElement {
- type_: Some(physical_type.into()),
- type_length: if type_length >= 0 {
- Some(type_length)
- } else {
- None
- },
- repetition_type: Some(basic_info.repetition().into()),
- name: basic_info.name().to_owned(),
- num_children: None,
- converted_type: basic_info.converted_type().into(),
- scale: if scale >= 0 { Some(scale) } else { None },
- precision: if precision >= 0 {
- Some(precision)
- } else {
- None
- },
- field_id: if basic_info.has_id() {
- Some(basic_info.id())
- } else {
- None
- },
- logical_type: basic_info.logical_type().map(|value| value.into()),
- };
-
- elements.push(element);
- }
- Type::GroupType {
- ref basic_info,
- ref fields,
- } => {
- let repetition = if basic_info.has_repetition() {
- Some(basic_info.repetition().into())
- } else {
- None
- };
-
- let element = SchemaElement {
- type_: None,
- type_length: None,
- repetition_type: repetition,
- name: basic_info.name().to_owned(),
- num_children: Some(fields.len() as i32),
- converted_type: basic_info.converted_type().into(),
- scale: None,
- precision: None,
- field_id: if basic_info.has_id() {
- Some(basic_info.id())
- } else {
- None
- },
- logical_type: basic_info.logical_type().map(|value| value.into()),
- };
-
- elements.push(element);
-
- // Add child elements for a group
- for field in fields {
- to_thrift_helper(field, elements);
- }
- }
- }
-}
-
-#[cfg(test)]
-mod tests {
- use super::*;
-
- use crate::basic::{DecimalType, IntType};
- use crate::schema::parser::parse_message_type;
-
- // TODO: add tests for v2 types
-
- #[test]
- fn test_primitive_type() {
- let mut result = Type::primitive_type_builder("foo", PhysicalType::INT32)
- .with_logical_type(Some(LogicalType::INTEGER(IntType {
- bit_width: 32,
- is_signed: true,
- })))
- .with_id(0)
- .build();
- assert!(result.is_ok());
-
- if let Ok(tp) = result {
- assert!(tp.is_primitive());
- assert!(!tp.is_group());
- let basic_info = tp.get_basic_info();
- assert_eq!(basic_info.repetition(), Repetition::OPTIONAL);
- assert_eq!(
- basic_info.logical_type(),
- Some(LogicalType::INTEGER(IntType {
- bit_width: 32,
- is_signed: true
- }))
- );
- assert_eq!(basic_info.converted_type(), ConvertedType::INT_32);
- assert_eq!(basic_info.id(), 0);
- match tp {
- Type::PrimitiveType { physical_type, .. } => {
- assert_eq!(physical_type, PhysicalType::INT32);
- }
- _ => panic!(),
- }
- }
-
- // Test illegal inputs with logical type
- result = Type::primitive_type_builder("foo", PhysicalType::INT64)
- .with_repetition(Repetition::REPEATED)
- .with_logical_type(Some(LogicalType::INTEGER(IntType {
- is_signed: true,
- bit_width: 8,
- })))
- .build();
- assert!(result.is_err());
- if let Err(e) = result {
- assert_eq!(
- format!("{}", e),
- "Parquet error: Cannot annotate INTEGER(IntType { bit_width: 8, is_signed: true }) from INT64 fields"
- );
- }
-
- // Test illegal inputs with converted type
- result = Type::primitive_type_builder("foo", PhysicalType::INT64)
- .with_repetition(Repetition::REPEATED)
- .with_converted_type(ConvertedType::BSON)
- .build();
- assert!(result.is_err());
- if let Err(e) = result {
- assert_eq!(
- format!("{}", e),
- "Parquet error: BSON can only annotate BYTE_ARRAY fields"
- );
- }
-
- result = Type::primitive_type_builder("foo", PhysicalType::INT96)
- .with_repetition(Repetition::REQUIRED)
- .with_converted_type(ConvertedType::DECIMAL)
- .with_precision(-1)
- .with_scale(-1)
- .build();
- assert!(result.is_err());
- if let Err(e) = result {
- assert_eq!(
- format!("{}", e),
- "Parquet error: DECIMAL can only annotate INT32, INT64, BYTE_ARRAY and FIXED_LEN_BYTE_ARRAY"
- );
- }
-
- result = Type::primitive_type_builder("foo", PhysicalType::BYTE_ARRAY)
- .with_repetition(Repetition::REQUIRED)
- .with_logical_type(Some(LogicalType::DECIMAL(DecimalType {
- scale: 32,
- precision: 12,
- })))
- .with_precision(-1)
- .with_scale(-1)
- .build();
- assert!(result.is_err());
- if let Err(e) = result {
- assert_eq!(
- format!("{}", e),
- "Parquet error: DECIMAL logical type scale 32 must match self.scale -1"
- );
- }
-
- result = Type::primitive_type_builder("foo", PhysicalType::BYTE_ARRAY)
- .with_repetition(Repetition::REQUIRED)
- .with_converted_type(ConvertedType::DECIMAL)
- .with_precision(-1)
- .with_scale(-1)
- .build();
- assert!(result.is_err());
- if let Err(e) = result {
- assert_eq!(
- format!("{}", e),
- "Parquet error: Invalid DECIMAL precision: -1"
- );
- }
-
- result = Type::primitive_type_builder("foo", PhysicalType::BYTE_ARRAY)
- .with_repetition(Repetition::REQUIRED)
- .with_converted_type(ConvertedType::DECIMAL)
- .with_precision(0)
- .with_scale(-1)
- .build();
- assert!(result.is_err());
- if let Err(e) = result {
- assert_eq!(
- format!("{}", e),
- "Parquet error: Invalid DECIMAL precision: 0"
- );
- }
-
- result = Type::primitive_type_builder("foo", PhysicalType::BYTE_ARRAY)
- .with_repetition(Repetition::REQUIRED)
- .with_converted_type(ConvertedType::DECIMAL)
- .with_precision(1)
- .with_scale(-1)
- .build();
- assert!(result.is_err());
- if let Err(e) = result {
- assert_eq!(format!("{}", e), "Parquet error: Invalid DECIMAL scale: -1");
- }
-
- result = Type::primitive_type_builder("foo", PhysicalType::BYTE_ARRAY)
- .with_repetition(Repetition::REQUIRED)
- .with_converted_type(ConvertedType::DECIMAL)
- .with_precision(1)
- .with_scale(2)
- .build();
- assert!(result.is_err());
- if let Err(e) = result {
- assert_eq!(
- format!("{}", e),
- "Parquet error: Invalid DECIMAL: scale (2) cannot be greater than or equal to precision (1)"
- );
- }
-
- result = Type::primitive_type_builder("foo", PhysicalType::INT32)
- .with_repetition(Repetition::REQUIRED)
- .with_converted_type(ConvertedType::DECIMAL)
- .with_precision(18)
- .with_scale(2)
- .build();
- assert!(result.is_err());
- if let Err(e) = result {
- assert_eq!(
- format!("{}", e),
- "Parquet error: Cannot represent INT32 as DECIMAL with precision 18"
- );
- }
-
- result = Type::primitive_type_builder("foo", PhysicalType::INT64)
- .with_repetition(Repetition::REQUIRED)
- .with_converted_type(ConvertedType::DECIMAL)
- .with_precision(32)
- .with_scale(2)
- .build();
- assert!(result.is_err());
- if let Err(e) = result {
- assert_eq!(
- format!("{}", e),
- "Parquet error: Cannot represent INT64 as DECIMAL with precision 32"
- );
- }
-
- result = Type::primitive_type_builder("foo", PhysicalType::FIXED_LEN_BYTE_ARRAY)
- .with_repetition(Repetition::REQUIRED)
- .with_converted_type(ConvertedType::DECIMAL)
- .with_length(5)
- .with_precision(12)
- .with_scale(2)
- .build();
- assert!(result.is_err());
- if let Err(e) = result {
- assert_eq!(
- format!("{}", e),
- "Parquet error: Cannot represent FIXED_LEN_BYTE_ARRAY as DECIMAL with length 5 and precision 12. The max precision can only be 11"
- );
- }
-
- result = Type::primitive_type_builder("foo", PhysicalType::INT64)
- .with_repetition(Repetition::REQUIRED)
- .with_converted_type(ConvertedType::UINT_8)
- .build();
- assert!(result.is_err());
- if let Err(e) = result {
- assert_eq!(
- format!("{}", e),
- "Parquet error: UINT_8 can only annotate INT32"
- );
- }
-
- result = Type::primitive_type_builder("foo", PhysicalType::INT32)
- .with_repetition(Repetition::REQUIRED)
- .with_converted_type(ConvertedType::TIME_MICROS)
- .build();
- assert!(result.is_err());
- if let Err(e) = result {
- assert_eq!(
- format!("{}", e),
- "Parquet error: TIME_MICROS can only annotate INT64"
- );
- }
-
- result = Type::primitive_type_builder("foo", PhysicalType::BYTE_ARRAY)
- .with_repetition(Repetition::REQUIRED)
- .with_converted_type(ConvertedType::INTERVAL)
- .build();
- assert!(result.is_err());
- if let Err(e) = result {
- assert_eq!(
- format!("{}", e),
- "Parquet error: INTERVAL can only annotate FIXED_LEN_BYTE_ARRAY(12)"
- );
- }
-
- result = Type::primitive_type_builder("foo", PhysicalType::FIXED_LEN_BYTE_ARRAY)
- .with_repetition(Repetition::REQUIRED)
- .with_converted_type(ConvertedType::INTERVAL)
- .with_length(1)
- .build();
- assert!(result.is_err());
- if let Err(e) = result {
- assert_eq!(
- format!("{}", e),
- "Parquet error: INTERVAL can only annotate FIXED_LEN_BYTE_ARRAY(12)"
- );
- }
-
- result = Type::primitive_type_builder("foo", PhysicalType::INT32)
- .with_repetition(Repetition::REQUIRED)
- .with_converted_type(ConvertedType::ENUM)
- .build();
- assert!(result.is_err());
- if let Err(e) = result {
- assert_eq!(
- format!("{}", e),
- "Parquet error: ENUM can only annotate BYTE_ARRAY fields"
- );
- }
-
- result = Type::primitive_type_builder("foo", PhysicalType::INT32)
- .with_repetition(Repetition::REQUIRED)
- .with_converted_type(ConvertedType::MAP)
- .build();
- assert!(result.is_err());
- if let Err(e) = result {
- assert_eq!(
- format!("{}", e),
- "Parquet error: MAP cannot be applied to a primitive type"
- );
- }
-
- result = Type::primitive_type_builder("foo", PhysicalType::FIXED_LEN_BYTE_ARRAY)
- .with_repetition(Repetition::REQUIRED)
- .with_converted_type(ConvertedType::DECIMAL)
- .with_length(-1)
- .build();
- assert!(result.is_err());
- if let Err(e) = result {
- assert_eq!(
- format!("{}", e),
- "Parquet error: Invalid FIXED_LEN_BYTE_ARRAY length: -1"
- );
- }
- }
-
- #[test]
- fn test_group_type() {
- let f1 = Type::primitive_type_builder("f1", PhysicalType::INT32)
- .with_converted_type(ConvertedType::INT_32)
- .with_id(0)
- .build();
- assert!(f1.is_ok());
- let f2 = Type::primitive_type_builder("f2", PhysicalType::BYTE_ARRAY)
- .with_converted_type(ConvertedType::UTF8)
- .with_id(1)
- .build();
- assert!(f2.is_ok());
-
- let mut fields = vec![];
- fields.push(Arc::new(f1.unwrap()));
- fields.push(Arc::new(f2.unwrap()));
-
- let result = Type::group_type_builder("foo")
- .with_repetition(Repetition::REPEATED)
- .with_logical_type(Some(LogicalType::LIST(Default::default())))
- .with_fields(&mut fields)
- .with_id(1)
- .build();
- assert!(result.is_ok());
-
- let tp = result.unwrap();
- let basic_info = tp.get_basic_info();
- assert!(tp.is_group());
- assert!(!tp.is_primitive());
- assert_eq!(basic_info.repetition(), Repetition::REPEATED);
- assert_eq!(
- basic_info.logical_type(),
- Some(LogicalType::LIST(Default::default()))
- );
- assert_eq!(basic_info.converted_type(), ConvertedType::LIST);
- assert_eq!(basic_info.id(), 1);
- assert_eq!(tp.get_fields().len(), 2);
- assert_eq!(tp.get_fields()[0].name(), "f1");
- assert_eq!(tp.get_fields()[1].name(), "f2");
- }
-
- #[test]
- fn test_column_descriptor() {
- let result = test_column_descriptor_helper();
- assert!(
- result.is_ok(),
- "Expected result to be OK but got err:\n {}",
- result.unwrap_err()
- );
- }
-
- fn test_column_descriptor_helper() -> Result<()> {
- let tp = Type::primitive_type_builder("name", PhysicalType::BYTE_ARRAY)
- .with_converted_type(ConvertedType::UTF8)
- .build()?;
-
- let descr = ColumnDescriptor::new(Arc::new(tp), 4, 1, ColumnPath::from("name"));
-
- assert_eq!(descr.path(), &ColumnPath::from("name"));
- assert_eq!(descr.converted_type(), ConvertedType::UTF8);
- assert_eq!(descr.physical_type(), PhysicalType::BYTE_ARRAY);
- assert_eq!(descr.max_def_level(), 4);
- assert_eq!(descr.max_rep_level(), 1);
- assert_eq!(descr.name(), "name");
- assert_eq!(descr.type_length(), -1);
- assert_eq!(descr.type_precision(), -1);
- assert_eq!(descr.type_scale(), -1);
-
- Ok(())
- }
-
- #[test]
- fn test_schema_descriptor() {
- let result = test_schema_descriptor_helper();
- assert!(
- result.is_ok(),
- "Expected result to be OK but got err:\n {}",
- result.unwrap_err()
- );
- }
-
- // A helper fn to avoid handling the results from type creation
- fn test_schema_descriptor_helper() -> Result<()> {
- let mut fields = vec![];
-
- let inta = Type::primitive_type_builder("a", PhysicalType::INT32)
- .with_repetition(Repetition::REQUIRED)
- .with_converted_type(ConvertedType::INT_32)
- .build()?;
- fields.push(Arc::new(inta));
- let intb = Type::primitive_type_builder("b", PhysicalType::INT64)
- .with_converted_type(ConvertedType::INT_64)
- .build()?;
- fields.push(Arc::new(intb));
- let intc = Type::primitive_type_builder("c", PhysicalType::BYTE_ARRAY)
- .with_repetition(Repetition::REPEATED)
- .with_converted_type(ConvertedType::UTF8)
- .build()?;
- fields.push(Arc::new(intc));
-
- // 3-level list encoding
- let item1 = Type::primitive_type_builder("item1", PhysicalType::INT64)
- .with_repetition(Repetition::REQUIRED)
- .with_converted_type(ConvertedType::INT_64)
- .build()?;
- let item2 =
- Type::primitive_type_builder("item2", PhysicalType::BOOLEAN).build()?;
- let item3 = Type::primitive_type_builder("item3", PhysicalType::INT32)
- .with_repetition(Repetition::REPEATED)
- .with_converted_type(ConvertedType::INT_32)
- .build()?;
- let list = Type::group_type_builder("records")
- .with_repetition(Repetition::REPEATED)
- .with_converted_type(ConvertedType::LIST)
- .with_fields(&mut vec![Arc::new(item1), Arc::new(item2), Arc::new(item3)])
- .build()?;
- let bag = Type::group_type_builder("bag")
- .with_repetition(Repetition::OPTIONAL)
- .with_fields(&mut vec![Arc::new(list)])
- .build()?;
- fields.push(Arc::new(bag));
-
- let schema = Type::group_type_builder("schema")
- .with_repetition(Repetition::REPEATED)
- .with_fields(&mut fields)
- .build()?;
- let descr = SchemaDescriptor::new(Arc::new(schema));
-
- let nleaves = 6;
- assert_eq!(descr.num_columns(), nleaves);
-
- // mdef mrep
- // required int32 a 0 0
- // optional int64 b 1 0
- // repeated byte_array c 1 1
- // optional group bag 1 0
- // repeated group records 2 1
- // required int64 item1 2 1
- // optional boolean item2 3 1
- // repeated int32 item3 3 2
- let ex_max_def_levels = vec![0, 1, 1, 2, 3, 3];
- let ex_max_rep_levels = vec![0, 0, 1, 1, 1, 2];
-
- for i in 0..nleaves {
- let col = descr.column(i);
- assert_eq!(col.max_def_level(), ex_max_def_levels[i], "{}", i);
- assert_eq!(col.max_rep_level(), ex_max_rep_levels[i], "{}", i);
- }
-
- assert_eq!(descr.column(0).path().string(), "a");
- assert_eq!(descr.column(1).path().string(), "b");
- assert_eq!(descr.column(2).path().string(), "c");
- assert_eq!(descr.column(3).path().string(), "bag.records.item1");
- assert_eq!(descr.column(4).path().string(), "bag.records.item2");
- assert_eq!(descr.column(5).path().string(), "bag.records.item3");
-
- assert_eq!(descr.get_column_root(0).name(), "a");
- assert_eq!(descr.get_column_root(3).name(), "bag");
- assert_eq!(descr.get_column_root(4).name(), "bag");
- assert_eq!(descr.get_column_root(5).name(), "bag");
-
- Ok(())
- }
-
- #[test]
- fn test_schema_build_tree_def_rep_levels() {
- let message_type = "
- message spark_schema {
- REQUIRED INT32 a;
- OPTIONAL group b {
- OPTIONAL INT32 _1;
- OPTIONAL INT32 _2;
- }
- OPTIONAL group c (LIST) {
- REPEATED group list {
- OPTIONAL INT32 element;
- }
- }
- }
- ";
- let schema = parse_message_type(message_type).expect("should parse schema");
- let descr = SchemaDescriptor::new(Arc::new(schema));
- // required int32 a
- assert_eq!(descr.column(0).max_def_level(), 0);
- assert_eq!(descr.column(0).max_rep_level(), 0);
- // optional int32 b._1
- assert_eq!(descr.column(1).max_def_level(), 2);
- assert_eq!(descr.column(1).max_rep_level(), 0);
- // optional int32 b._2
- assert_eq!(descr.column(2).max_def_level(), 2);
- assert_eq!(descr.column(2).max_rep_level(), 0);
- // repeated optional int32 c.list.element
- assert_eq!(descr.column(3).max_def_level(), 3);
- assert_eq!(descr.column(3).max_rep_level(), 1);
- }
-
- #[test]
- #[should_panic(expected = "Cannot call get_physical_type() on a non-primitive type")]
- fn test_get_physical_type_panic() {
- let list = Type::group_type_builder("records")
- .with_repetition(Repetition::REPEATED)
- .build()
- .unwrap();
- list.get_physical_type();
- }
-
- #[test]
- fn test_get_physical_type_primitive() {
- let f = Type::primitive_type_builder("f", PhysicalType::INT64)
- .build()
- .unwrap();
- assert_eq!(f.get_physical_type(), PhysicalType::INT64);
-
- let f = Type::primitive_type_builder("f", PhysicalType::BYTE_ARRAY)
- .build()
- .unwrap();
- assert_eq!(f.get_physical_type(), PhysicalType::BYTE_ARRAY);
- }
-
- #[test]
- fn test_check_contains_primitive_primitive() {
- // OK
- let f1 = Type::primitive_type_builder("f", PhysicalType::INT32)
- .build()
- .unwrap();
- let f2 = Type::primitive_type_builder("f", PhysicalType::INT32)
- .build()
- .unwrap();
- assert!(f1.check_contains(&f2));
-
- // OK: different logical type does not affect check_contains
- let f1 = Type::primitive_type_builder("f", PhysicalType::INT32)
- .with_converted_type(ConvertedType::UINT_8)
- .build()
- .unwrap();
- let f2 = Type::primitive_type_builder("f", PhysicalType::INT32)
- .with_converted_type(ConvertedType::UINT_16)
- .build()
- .unwrap();
- assert!(f1.check_contains(&f2));
-
- // KO: different name
- let f1 = Type::primitive_type_builder("f1", PhysicalType::INT32)
- .build()
- .unwrap();
- let f2 = Type::primitive_type_builder("f2", PhysicalType::INT32)
- .build()
- .unwrap();
- assert!(!f1.check_contains(&f2));
-
- // KO: different type
- let f1 = Type::primitive_type_builder("f", PhysicalType::INT32)
- .build()
- .unwrap();
- let f2 = Type::primitive_type_builder("f", PhysicalType::INT64)
- .build()
- .unwrap();
- assert!(!f1.check_contains(&f2));
-
- // KO: different repetition
- let f1 = Type::primitive_type_builder("f", PhysicalType::INT32)
- .with_repetition(Repetition::REQUIRED)
- .build()
- .unwrap();
- let f2 = Type::primitive_type_builder("f", PhysicalType::INT32)
- .with_repetition(Repetition::OPTIONAL)
- .build()
- .unwrap();
- assert!(!f1.check_contains(&f2));
- }
-
- // function to create a new group type for testing
- fn test_new_group_type(name: &str, repetition: Repetition, types: Vec<Type>) -> Type {
- let mut fields = Vec::new();
- for tpe in types {
- fields.push(Arc::new(tpe))
- }
- Type::group_type_builder(name)
- .with_repetition(repetition)
- .with_fields(&mut fields)
- .build()
- .unwrap()
- }
-
- #[test]
- fn test_check_contains_group_group() {
- // OK: should match okay with empty fields
- let f1 = Type::group_type_builder("f").build().unwrap();
- let f2 = Type::group_type_builder("f").build().unwrap();
- assert!(f1.check_contains(&f2));
-
- // OK: fields match
- let f1 = test_new_group_type(
- "f",
- Repetition::REPEATED,
- vec![
- Type::primitive_type_builder("f1", PhysicalType::INT32)
- .build()
- .unwrap(),
- Type::primitive_type_builder("f2", PhysicalType::INT64)
- .build()
- .unwrap(),
- ],
- );
- let f2 = test_new_group_type(
- "f",
- Repetition::REPEATED,
- vec![
- Type::primitive_type_builder("f1", PhysicalType::INT32)
- .build()
- .unwrap(),
- Type::primitive_type_builder("f2", PhysicalType::INT64)
- .build()
- .unwrap(),
- ],
- );
- assert!(f1.check_contains(&f2));
-
- // OK: subset of fields
- let f1 = test_new_group_type(
- "f",
- Repetition::REPEATED,
- vec![
- Type::primitive_type_builder("f1", PhysicalType::INT32)
- .build()
- .unwrap(),
- Type::primitive_type_builder("f2", PhysicalType::INT64)
- .build()
- .unwrap(),
- ],
- );
- let f2 = test_new_group_type(
- "f",
- Repetition::REPEATED,
- vec![Type::primitive_type_builder("f2", PhysicalType::INT64)
- .build()
- .unwrap()],
- );
- assert!(f1.check_contains(&f2));
-
- // KO: different name
- let f1 = Type::group_type_builder("f1").build().unwrap();
- let f2 = Type::group_type_builder("f2").build().unwrap();
- assert!(!f1.check_contains(&f2));
-
- // KO: different repetition
- let f1 = Type::group_type_builder("f")
- .with_repetition(Repetition::OPTIONAL)
- .build()
- .unwrap();
- let f2 = Type::group_type_builder("f")
- .with_repetition(Repetition::REPEATED)
- .build()
- .unwrap();
- assert!(!f1.check_contains(&f2));
-
- // KO: different fields
- let f1 = test_new_group_type(
- "f",
- Repetition::REPEATED,
- vec![
- Type::primitive_type_builder("f1", PhysicalType::INT32)
- .build()
- .unwrap(),
- Type::primitive_type_builder("f2", PhysicalType::INT64)
- .build()
- .unwrap(),
- ],
- );
- let f2 = test_new_group_type(
- "f",
- Repetition::REPEATED,
- vec![
- Type::primitive_type_builder("f1", PhysicalType::INT32)
- .build()
- .unwrap(),
- Type::primitive_type_builder("f2", PhysicalType::BOOLEAN)
- .build()
- .unwrap(),
- ],
- );
- assert!(!f1.check_contains(&f2));
-
- // KO: different fields
- let f1 = test_new_group_type(
- "f",
- Repetition::REPEATED,
- vec![
- Type::primitive_type_builder("f1", PhysicalType::INT32)
- .build()
- .unwrap(),
- Type::primitive_type_builder("f2", PhysicalType::INT64)
- .build()
- .unwrap(),
- ],
- );
- let f2 = test_new_group_type(
- "f",
- Repetition::REPEATED,
- vec![Type::primitive_type_builder("f3", PhysicalType::INT32)
- .build()
- .unwrap()],
- );
- assert!(!f1.check_contains(&f2));
- }
-
- #[test]
- fn test_check_contains_group_primitive() {
- // KO: should not match
- let f1 = Type::group_type_builder("f").build().unwrap();
- let f2 = Type::primitive_type_builder("f", PhysicalType::INT64)
- .build()
- .unwrap();
- assert!(!f1.check_contains(&f2));
- assert!(!f2.check_contains(&f1));
-
- // KO: should not match when primitive field is part of group type
- let f1 = test_new_group_type(
- "f",
- Repetition::REPEATED,
- vec![Type::primitive_type_builder("f1", PhysicalType::INT32)
- .build()
- .unwrap()],
- );
- let f2 = Type::primitive_type_builder("f1", PhysicalType::INT32)
- .build()
- .unwrap();
- assert!(!f1.check_contains(&f2));
- assert!(!f2.check_contains(&f1));
-
- // OK: match nested types
- let f1 = test_new_group_type(
- "a",
- Repetition::REPEATED,
- vec![
- test_new_group_type(
- "b",
- Repetition::REPEATED,
- vec![Type::primitive_type_builder("c", PhysicalType::INT32)
- .build()
- .unwrap()],
- ),
- Type::primitive_type_builder("d", PhysicalType::INT64)
- .build()
- .unwrap(),
- Type::primitive_type_builder("e", PhysicalType::BOOLEAN)
- .build()
- .unwrap(),
- ],
- );
- let f2 = test_new_group_type(
- "a",
- Repetition::REPEATED,
- vec![test_new_group_type(
- "b",
- Repetition::REPEATED,
- vec![Type::primitive_type_builder("c", PhysicalType::INT32)
- .build()
- .unwrap()],
- )],
- );
- assert!(f1.check_contains(&f2)); // should match
- assert!(!f2.check_contains(&f1)); // should fail
- }
-
- #[test]
- fn test_schema_type_thrift_conversion_err() {
- let schema = Type::primitive_type_builder("col", PhysicalType::INT32)
- .build()
- .unwrap();
- let thrift_schema = to_thrift(&schema);
- assert!(thrift_schema.is_err());
- if let Err(e) = thrift_schema {
- assert_eq!(
- format!("{}", e),
- "Parquet error: Root schema must be Group type"
- );
- }
- }
-
- #[test]
- fn test_schema_type_thrift_conversion() {
- let message_type = "
- message conversions {
- REQUIRED INT64 id;
- OPTIONAL group int_array_Array (LIST) {
- REPEATED group list {
- OPTIONAL group element (LIST) {
- REPEATED group list {
- OPTIONAL INT32 element;
- }
- }
- }
- }
- OPTIONAL group int_map (MAP) {
- REPEATED group map (MAP_KEY_VALUE) {
- REQUIRED BYTE_ARRAY key (UTF8);
- OPTIONAL INT32 value;
- }
- }
- OPTIONAL group int_Map_Array (LIST) {
- REPEATED group list {
- OPTIONAL group g (MAP) {
- REPEATED group map (MAP_KEY_VALUE) {
- REQUIRED BYTE_ARRAY key (UTF8);
- OPTIONAL group value {
- OPTIONAL group H {
- OPTIONAL group i (LIST) {
- REPEATED group list {
- OPTIONAL DOUBLE element;
- }
- }
- }
- }
- }
- }
- }
- }
- OPTIONAL group nested_struct {
- OPTIONAL INT32 A;
- OPTIONAL group b (LIST) {
- REPEATED group list {
- REQUIRED FIXED_LEN_BYTE_ARRAY (16) element;
- }
- }
- }
- }
- ";
- let expected_schema = parse_message_type(message_type).unwrap();
- let thrift_schema = to_thrift(&expected_schema).unwrap();
- let result_schema = from_thrift(&thrift_schema).unwrap();
- assert_eq!(result_schema, Arc::new(expected_schema));
- }
-
- #[test]
- fn test_schema_type_thrift_conversion_decimal() {
- let message_type = "
- message decimals {
- OPTIONAL INT32 field0;
- OPTIONAL INT64 field1 (DECIMAL (18, 2));
- OPTIONAL FIXED_LEN_BYTE_ARRAY (16) field2 (DECIMAL (38, 18));
- OPTIONAL BYTE_ARRAY field3 (DECIMAL (9));
- }
- ";
- let expected_schema = parse_message_type(message_type).unwrap();
- let thrift_schema = to_thrift(&expected_schema).unwrap();
- let result_schema = from_thrift(&thrift_schema).unwrap();
- assert_eq!(result_schema, Arc::new(expected_schema));
- }
-
- // Tests schema conversion from thrift, when num_children is set to Some(0) for a
- // primitive type.
- #[test]
- fn test_schema_from_thrift_with_num_children_set() {
- // schema definition written by parquet-cpp version 1.3.2-SNAPSHOT
- let message_type = "
- message schema {
- OPTIONAL BYTE_ARRAY id (UTF8);
- OPTIONAL BYTE_ARRAY name (UTF8);
- OPTIONAL BYTE_ARRAY message (UTF8);
- OPTIONAL INT32 type (UINT_8);
- OPTIONAL INT64 author_time (TIMESTAMP_MILLIS);
- OPTIONAL INT64 __index_level_0__;
- }
- ";
-
- let expected_schema = parse_message_type(message_type).unwrap();
- let mut thrift_schema = to_thrift(&expected_schema).unwrap();
- // Change all of None to Some(0)
- for mut elem in &mut thrift_schema[..] {
- if elem.num_children == None {
- elem.num_children = Some(0);
- }
- }
-
- let result_schema = from_thrift(&thrift_schema).unwrap();
- assert_eq!(result_schema, Arc::new(expected_schema));
- }
-
- // Sometimes parquet-cpp sets repetition level for the root node, which is against
- // the format definition, but we need to handle it by setting it back to None.
- #[test]
- fn test_schema_from_thrift_root_has_repetition() {
- // schema definition written by parquet-cpp version 1.3.2-SNAPSHOT
- let message_type = "
- message schema {
- OPTIONAL BYTE_ARRAY a (UTF8);
- OPTIONAL INT32 b (UINT_8);
- }
- ";
-
- let expected_schema = parse_message_type(message_type).unwrap();
- let mut thrift_schema = to_thrift(&expected_schema).unwrap();
- thrift_schema[0].repetition_type = Some(Repetition::REQUIRED.into());
-
- let result_schema = from_thrift(&thrift_schema).unwrap();
- assert_eq!(result_schema, Arc::new(expected_schema));
- }
-}
diff --git a/parquet/src/schema/visitor.rs b/parquet/src/schema/visitor.rs
deleted file mode 100644
index 61bc3be..0000000
--- a/parquet/src/schema/visitor.rs
+++ /dev/null
@@ -1,240 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-use crate::basic::{ConvertedType, Repetition};
-use crate::errors::ParquetError::General;
-use crate::errors::Result;
-use crate::schema::types::{Type, TypePtr};
-
-/// A utility trait to help user to traverse against parquet type.
-pub trait TypeVisitor<R, C> {
- /// Called when a primitive type hit.
- fn visit_primitive(&mut self, primitive_type: TypePtr, context: C) -> Result<R>;
-
- /// Default implementation when visiting a list.
- ///
- /// It checks list type definition and calls `visit_list_with_item` with extracted
- /// item type.
- ///
- /// To fully understand this algorithm, please refer to
- /// [parquet doc](https://github.com/apache/parquet-format/blob/master/LogicalTypes.md).
- fn visit_list(&mut self, list_type: TypePtr, context: C) -> Result<R> {
- match list_type.as_ref() {
- Type::PrimitiveType { .. } => panic!(
- "{:?} is a list type and can't be processed as primitive.",
- list_type
- ),
- Type::GroupType {
- basic_info: _,
- fields,
- } if fields.len() == 1 => {
- let list_item = fields.first().unwrap();
-
- match list_item.as_ref() {
- Type::PrimitiveType { .. } => {
- if list_item.get_basic_info().repetition() == Repetition::REPEATED
- {
- self.visit_list_with_item(
- list_type.clone(),
- list_item.clone(),
- context,
- )
- } else {
- Err(General(
- "Primitive element type of list must be repeated."
- .to_string(),
- ))
- }
- }
- Type::GroupType {
- basic_info: _,
- fields,
- } => {
- if fields.len() == 1
- && list_item.name() != "array"
- && list_item.name() != format!("{}_tuple", list_type.name())
- {
- self.visit_list_with_item(
- list_type.clone(),
- fields.first().unwrap().clone(),
- context,
- )
- } else {
- self.visit_list_with_item(
- list_type.clone(),
- list_item.clone(),
- context,
- )
- }
- }
- }
- }
- _ => Err(General(
- "Group element type of list can only contain one field.".to_string(),
- )),
- }
- }
-
- /// Called when a struct type hit.
- fn visit_struct(&mut self, struct_type: TypePtr, context: C) -> Result<R>;
-
- /// Called when a map type hit.
- fn visit_map(&mut self, map_type: TypePtr, context: C) -> Result<R>;
-
- /// A utility method which detects input type and calls corresponding method.
- fn dispatch(&mut self, cur_type: TypePtr, context: C) -> Result<R> {
- if cur_type.is_primitive() {
- self.visit_primitive(cur_type, context)
- } else {
- match cur_type.get_basic_info().converted_type() {
- ConvertedType::LIST => self.visit_list(cur_type, context),
- ConvertedType::MAP | ConvertedType::MAP_KEY_VALUE => {
- self.visit_map(cur_type, context)
- }
- _ => self.visit_struct(cur_type, context),
- }
- }
- }
-
- /// Called by `visit_list`.
- fn visit_list_with_item(
- &mut self,
- list_type: TypePtr,
- item_type: TypePtr,
- context: C,
- ) -> Result<R>;
-}
-
-#[cfg(test)]
-mod tests {
- use super::TypeVisitor;
- use crate::basic::Type as PhysicalType;
- use crate::errors::Result;
- use crate::schema::parser::parse_message_type;
- use crate::schema::types::TypePtr;
- use std::sync::Arc;
-
- struct TestVisitorContext {}
- struct TestVisitor {
- primitive_visited: bool,
- struct_visited: bool,
- list_visited: bool,
- root_type: TypePtr,
- }
-
- impl TypeVisitor<bool, TestVisitorContext> for TestVisitor {
- fn visit_primitive(
- &mut self,
- primitive_type: TypePtr,
- _context: TestVisitorContext,
- ) -> Result<bool> {
- assert_eq!(
- self.get_field_by_name(primitive_type.name()).as_ref(),
- primitive_type.as_ref()
- );
- self.primitive_visited = true;
- Ok(true)
- }
-
- fn visit_struct(
- &mut self,
- struct_type: TypePtr,
- _context: TestVisitorContext,
- ) -> Result<bool> {
- assert_eq!(
- self.get_field_by_name(struct_type.name()).as_ref(),
- struct_type.as_ref()
- );
- self.struct_visited = true;
- Ok(true)
- }
-
- fn visit_map(
- &mut self,
- _map_type: TypePtr,
- _context: TestVisitorContext,
- ) -> Result<bool> {
- unimplemented!()
- }
-
- fn visit_list_with_item(
- &mut self,
- list_type: TypePtr,
- item_type: TypePtr,
- _context: TestVisitorContext,
- ) -> Result<bool> {
- assert_eq!(
- self.get_field_by_name(list_type.name()).as_ref(),
- list_type.as_ref()
- );
- assert_eq!("element", item_type.name());
- assert_eq!(PhysicalType::INT32, item_type.get_physical_type());
- self.list_visited = true;
- Ok(true)
- }
- }
-
- impl TestVisitor {
- fn new(root: TypePtr) -> Self {
- Self {
- primitive_visited: false,
- struct_visited: false,
- list_visited: false,
- root_type: root,
- }
- }
-
- fn get_field_by_name(&self, name: &str) -> TypePtr {
- self.root_type
- .get_fields()
- .iter()
- .find(|t| t.name() == name)
- .cloned()
- .unwrap()
- }
- }
-
- #[test]
- fn test_visitor() {
- let message_type = "
- message spark_schema {
- REQUIRED INT32 a;
- OPTIONAL group inner_schema {
- REQUIRED INT32 b;
- REQUIRED DOUBLE c;
- }
-
- OPTIONAL group e (LIST) {
- REPEATED group list {
- REQUIRED INT32 element;
- }
- }
- ";
-
- let parquet_type = Arc::new(parse_message_type(&message_type).unwrap());
-
- let mut visitor = TestVisitor::new(parquet_type.clone());
- for f in parquet_type.get_fields() {
- let c = TestVisitorContext {};
- assert!(visitor.dispatch(f.clone(), c).unwrap());
- }
-
- assert!(visitor.struct_visited);
- assert!(visitor.primitive_visited);
- assert!(visitor.list_visited);
- }
-}
diff --git a/parquet/src/util/bit_packing.rs b/parquet/src/util/bit_packing.rs
deleted file mode 100644
index 6b9673f..0000000
--- a/parquet/src/util/bit_packing.rs
+++ /dev/null
@@ -1,3662 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-/// Unpack 32 values with bit width `num_bits` from `in_ptr`, and write to `out_ptr`.
-/// Return the `in_ptr` where the starting offset points to the first byte after all the
-/// bytes that were consumed.
-// TODO: may be better to make these more compact using if-else conditions.
-// However, this may require const generics:
-// https://github.com/rust-lang/rust/issues/44580
-// to eliminate the branching cost.
-// TODO: we should use SIMD instructions to further optimize this. I have explored
-// https://github.com/tantivy-search/bitpacking
-// but the layout it uses for SIMD is different from Parquet.
-// TODO: support packing as well, which is used for encoding.
-pub unsafe fn unpack32(
- mut in_ptr: *const u32,
- out_ptr: *mut u32,
- num_bits: usize,
-) -> *const u32 {
- in_ptr = match num_bits {
- 0 => nullunpacker32(in_ptr, out_ptr),
- 1 => unpack1_32(in_ptr, out_ptr),
- 2 => unpack2_32(in_ptr, out_ptr),
- 3 => unpack3_32(in_ptr, out_ptr),
- 4 => unpack4_32(in_ptr, out_ptr),
- 5 => unpack5_32(in_ptr, out_ptr),
- 6 => unpack6_32(in_ptr, out_ptr),
- 7 => unpack7_32(in_ptr, out_ptr),
- 8 => unpack8_32(in_ptr, out_ptr),
- 9 => unpack9_32(in_ptr, out_ptr),
- 10 => unpack10_32(in_ptr, out_ptr),
- 11 => unpack11_32(in_ptr, out_ptr),
- 12 => unpack12_32(in_ptr, out_ptr),
- 13 => unpack13_32(in_ptr, out_ptr),
- 14 => unpack14_32(in_ptr, out_ptr),
- 15 => unpack15_32(in_ptr, out_ptr),
- 16 => unpack16_32(in_ptr, out_ptr),
- 17 => unpack17_32(in_ptr, out_ptr),
- 18 => unpack18_32(in_ptr, out_ptr),
- 19 => unpack19_32(in_ptr, out_ptr),
- 20 => unpack20_32(in_ptr, out_ptr),
- 21 => unpack21_32(in_ptr, out_ptr),
- 22 => unpack22_32(in_ptr, out_ptr),
- 23 => unpack23_32(in_ptr, out_ptr),
- 24 => unpack24_32(in_ptr, out_ptr),
- 25 => unpack25_32(in_ptr, out_ptr),
- 26 => unpack26_32(in_ptr, out_ptr),
- 27 => unpack27_32(in_ptr, out_ptr),
- 28 => unpack28_32(in_ptr, out_ptr),
- 29 => unpack29_32(in_ptr, out_ptr),
- 30 => unpack30_32(in_ptr, out_ptr),
- 31 => unpack31_32(in_ptr, out_ptr),
- 32 => unpack32_32(in_ptr, out_ptr),
- _ => unimplemented!(),
- };
- in_ptr
-}
-
-unsafe fn nullunpacker32(in_buf: *const u32, mut out: *mut u32) -> *const u32 {
- for _ in 0..32 {
- *out = 0;
- out = out.offset(1);
- }
- in_buf
-}
-
-unsafe fn unpack1_32(in_buf: *const u32, mut out: *mut u32) -> *const u32 {
- *out = (*in_buf) & 1;
- out = out.offset(1);
- *out = ((*in_buf) >> 1) & 1;
- out = out.offset(1);
- *out = ((*in_buf) >> 2) & 1;
- out = out.offset(1);
- *out = ((*in_buf) >> 3) & 1;
- out = out.offset(1);
- *out = ((*in_buf) >> 4) & 1;
- out = out.offset(1);
- *out = ((*in_buf) >> 5) & 1;
- out = out.offset(1);
- *out = ((*in_buf) >> 6) & 1;
- out = out.offset(1);
- *out = ((*in_buf) >> 7) & 1;
- out = out.offset(1);
- *out = ((*in_buf) >> 8) & 1;
- out = out.offset(1);
- *out = ((*in_buf) >> 9) & 1;
- out = out.offset(1);
- *out = ((*in_buf) >> 10) & 1;
- out = out.offset(1);
- *out = ((*in_buf) >> 11) & 1;
- out = out.offset(1);
- *out = ((*in_buf) >> 12) & 1;
- out = out.offset(1);
- *out = ((*in_buf) >> 13) & 1;
- out = out.offset(1);
- *out = ((*in_buf) >> 14) & 1;
- out = out.offset(1);
- *out = ((*in_buf) >> 15) & 1;
- out = out.offset(1);
- *out = ((*in_buf) >> 16) & 1;
- out = out.offset(1);
- *out = ((*in_buf) >> 17) & 1;
- out = out.offset(1);
- *out = ((*in_buf) >> 18) & 1;
- out = out.offset(1);
- *out = ((*in_buf) >> 19) & 1;
- out = out.offset(1);
- *out = ((*in_buf) >> 20) & 1;
- out = out.offset(1);
- *out = ((*in_buf) >> 21) & 1;
- out = out.offset(1);
- *out = ((*in_buf) >> 22) & 1;
- out = out.offset(1);
- *out = ((*in_buf) >> 23) & 1;
- out = out.offset(1);
- *out = ((*in_buf) >> 24) & 1;
- out = out.offset(1);
- *out = ((*in_buf) >> 25) & 1;
- out = out.offset(1);
- *out = ((*in_buf) >> 26) & 1;
- out = out.offset(1);
- *out = ((*in_buf) >> 27) & 1;
- out = out.offset(1);
- *out = ((*in_buf) >> 28) & 1;
- out = out.offset(1);
- *out = ((*in_buf) >> 29) & 1;
- out = out.offset(1);
- *out = ((*in_buf) >> 30) & 1;
- out = out.offset(1);
- *out = (*in_buf) >> 31;
-
- in_buf.offset(1)
-}
-
-unsafe fn unpack2_32(mut in_buf: *const u32, mut out: *mut u32) -> *const u32 {
- *out = (*in_buf) % (1u32 << 2);
- out = out.offset(1);
- *out = ((*in_buf) >> 2) % (1u32 << 2);
- out = out.offset(1);
- *out = ((*in_buf) >> 4) % (1u32 << 2);
- out = out.offset(1);
- *out = ((*in_buf) >> 6) % (1u32 << 2);
- out = out.offset(1);
- *out = ((*in_buf) >> 8) % (1u32 << 2);
- out = out.offset(1);
- *out = ((*in_buf) >> 10) % (1u32 << 2);
- out = out.offset(1);
- *out = ((*in_buf) >> 12) % (1u32 << 2);
- out = out.offset(1);
- *out = ((*in_buf) >> 14) % (1u32 << 2);
- out = out.offset(1);
- *out = ((*in_buf) >> 16) % (1u32 << 2);
- out = out.offset(1);
- *out = ((*in_buf) >> 18) % (1u32 << 2);
- out = out.offset(1);
- *out = ((*in_buf) >> 20) % (1u32 << 2);
- out = out.offset(1);
- *out = ((*in_buf) >> 22) % (1u32 << 2);
- out = out.offset(1);
- *out = ((*in_buf) >> 24) % (1u32 << 2);
- out = out.offset(1);
- *out = ((*in_buf) >> 26) % (1u32 << 2);
- out = out.offset(1);
- *out = ((*in_buf) >> 28) % (1u32 << 2);
- out = out.offset(1);
- *out = (*in_buf) >> 30;
- out = out.offset(1);
- in_buf = in_buf.offset(1);
- *out = (*in_buf) % (1u32 << 2);
- out = out.offset(1);
- *out = ((*in_buf) >> 2) % (1u32 << 2);
- out = out.offset(1);
- *out = ((*in_buf) >> 4) % (1u32 << 2);
- out = out.offset(1);
- *out = ((*in_buf) >> 6) % (1u32 << 2);
- out = out.offset(1);
- *out = ((*in_buf) >> 8) % (1u32 << 2);
- out = out.offset(1);
- *out = ((*in_buf) >> 10) % (1u32 << 2);
- out = out.offset(1);
- *out = ((*in_buf) >> 12) % (1u32 << 2);
- out = out.offset(1);
- *out = ((*in_buf) >> 14) % (1u32 << 2);
- out = out.offset(1);
- *out = ((*in_buf) >> 16) % (1u32 << 2);
- out = out.offset(1);
- *out = ((*in_buf) >> 18) % (1u32 << 2);
- out = out.offset(1);
- *out = ((*in_buf) >> 20) % (1u32 << 2);
- out = out.offset(1);
- *out = ((*in_buf) >> 22) % (1u32 << 2);
- out = out.offset(1);
- *out = ((*in_buf) >> 24) % (1u32 << 2);
- out = out.offset(1);
- *out = ((*in_buf) >> 26) % (1u32 << 2);
- out = out.offset(1);
- *out = ((*in_buf) >> 28) % (1u32 << 2);
- out = out.offset(1);
- *out = (*in_buf) >> 30;
-
- in_buf.offset(1)
-}
-
-unsafe fn unpack3_32(mut in_buf: *const u32, mut out: *mut u32) -> *const u32 {
- *out = (*in_buf) % (1u32 << 3);
- out = out.offset(1);
- *out = ((*in_buf) >> 3) % (1u32 << 3);
- out = out.offset(1);
- *out = ((*in_buf) >> 6) % (1u32 << 3);
- out = out.offset(1);
- *out = ((*in_buf) >> 9) % (1u32 << 3);
- out = out.offset(1);
- *out = ((*in_buf) >> 12) % (1u32 << 3);
- out = out.offset(1);
- *out = ((*in_buf) >> 15) % (1u32 << 3);
- out = out.offset(1);
- *out = ((*in_buf) >> 18) % (1u32 << 3);
- out = out.offset(1);
- *out = ((*in_buf) >> 21) % (1u32 << 3);
- out = out.offset(1);
- *out = ((*in_buf) >> 24) % (1u32 << 3);
- out = out.offset(1);
- *out = ((*in_buf) >> 27) % (1u32 << 3);
- out = out.offset(1);
- *out = (*in_buf) >> 30;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 1)) << (3 - 1);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 1) % (1u32 << 3);
- out = out.offset(1);
- *out = ((*in_buf) >> 4) % (1u32 << 3);
- out = out.offset(1);
- *out = ((*in_buf) >> 7) % (1u32 << 3);
- out = out.offset(1);
- *out = ((*in_buf) >> 10) % (1u32 << 3);
- out = out.offset(1);
- *out = ((*in_buf) >> 13) % (1u32 << 3);
- out = out.offset(1);
- *out = ((*in_buf) >> 16) % (1u32 << 3);
- out = out.offset(1);
- *out = ((*in_buf) >> 19) % (1u32 << 3);
- out = out.offset(1);
- *out = ((*in_buf) >> 22) % (1u32 << 3);
- out = out.offset(1);
- *out = ((*in_buf) >> 25) % (1u32 << 3);
- out = out.offset(1);
- *out = ((*in_buf) >> 28) % (1u32 << 3);
- out = out.offset(1);
- *out = (*in_buf) >> 31;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 2)) << (3 - 2);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 2) % (1u32 << 3);
- out = out.offset(1);
- *out = ((*in_buf) >> 5) % (1u32 << 3);
- out = out.offset(1);
- *out = ((*in_buf) >> 8) % (1u32 << 3);
- out = out.offset(1);
- *out = ((*in_buf) >> 11) % (1u32 << 3);
- out = out.offset(1);
- *out = ((*in_buf) >> 14) % (1u32 << 3);
- out = out.offset(1);
- *out = ((*in_buf) >> 17) % (1u32 << 3);
- out = out.offset(1);
- *out = ((*in_buf) >> 20) % (1u32 << 3);
- out = out.offset(1);
- *out = ((*in_buf) >> 23) % (1u32 << 3);
- out = out.offset(1);
- *out = ((*in_buf) >> 26) % (1u32 << 3);
- out = out.offset(1);
- *out = (*in_buf) >> 29;
-
- in_buf.offset(1)
-}
-
-unsafe fn unpack4_32(mut in_buf: *const u32, mut out: *mut u32) -> *const u32 {
- *out = (*in_buf) % (1u32 << 4);
- out = out.offset(1);
- *out = ((*in_buf) >> 4) % (1u32 << 4);
- out = out.offset(1);
- *out = ((*in_buf) >> 8) % (1u32 << 4);
- out = out.offset(1);
- *out = ((*in_buf) >> 12) % (1u32 << 4);
- out = out.offset(1);
- *out = ((*in_buf) >> 16) % (1u32 << 4);
- out = out.offset(1);
- *out = ((*in_buf) >> 20) % (1u32 << 4);
- out = out.offset(1);
- *out = ((*in_buf) >> 24) % (1u32 << 4);
- out = out.offset(1);
- *out = ((*in_buf) >> 28) % (1u32 << 4);
- out = out.offset(1);
- in_buf = in_buf.offset(1);
-
- *out = (*in_buf) % (1u32 << 4);
- out = out.offset(1);
- *out = ((*in_buf) >> 4) % (1u32 << 4);
- out = out.offset(1);
- *out = ((*in_buf) >> 8) % (1u32 << 4);
- out = out.offset(1);
- *out = ((*in_buf) >> 12) % (1u32 << 4);
- out = out.offset(1);
- *out = ((*in_buf) >> 16) % (1u32 << 4);
- out = out.offset(1);
- *out = ((*in_buf) >> 20) % (1u32 << 4);
- out = out.offset(1);
- *out = ((*in_buf) >> 24) % (1u32 << 4);
- out = out.offset(1);
- *out = ((*in_buf) >> 28) % (1u32 << 4);
- out = out.offset(1);
- in_buf = in_buf.offset(1);
-
- *out = (*in_buf) % (1u32 << 4);
- out = out.offset(1);
- *out = ((*in_buf) >> 4) % (1u32 << 4);
- out = out.offset(1);
- *out = ((*in_buf) >> 8) % (1u32 << 4);
- out = out.offset(1);
- *out = ((*in_buf) >> 12) % (1u32 << 4);
- out = out.offset(1);
- *out = ((*in_buf) >> 16) % (1u32 << 4);
- out = out.offset(1);
- *out = ((*in_buf) >> 20) % (1u32 << 4);
- out = out.offset(1);
- *out = ((*in_buf) >> 24) % (1u32 << 4);
- out = out.offset(1);
- *out = ((*in_buf) >> 28) % (1u32 << 4);
- out = out.offset(1);
- in_buf = in_buf.offset(1);
-
- *out = (*in_buf) % (1u32 << 4);
- out = out.offset(1);
- *out = ((*in_buf) >> 4) % (1u32 << 4);
- out = out.offset(1);
- *out = ((*in_buf) >> 8) % (1u32 << 4);
- out = out.offset(1);
- *out = ((*in_buf) >> 12) % (1u32 << 4);
- out = out.offset(1);
- *out = ((*in_buf) >> 16) % (1u32 << 4);
- out = out.offset(1);
- *out = ((*in_buf) >> 20) % (1u32 << 4);
- out = out.offset(1);
- *out = ((*in_buf) >> 24) % (1u32 << 4);
- out = out.offset(1);
- *out = ((*in_buf) >> 28) % (1u32 << 4);
-
- in_buf.offset(1)
-}
-
-unsafe fn unpack5_32(mut in_buf: *const u32, mut out: *mut u32) -> *const u32 {
- *out = (*in_buf) % (1u32 << 5);
- out = out.offset(1);
- *out = ((*in_buf) >> 5) % (1u32 << 5);
- out = out.offset(1);
- *out = ((*in_buf) >> 10) % (1u32 << 5);
- out = out.offset(1);
- *out = ((*in_buf) >> 15) % (1u32 << 5);
- out = out.offset(1);
- *out = ((*in_buf) >> 20) % (1u32 << 5);
- out = out.offset(1);
- *out = ((*in_buf) >> 25) % (1u32 << 5);
- out = out.offset(1);
- *out = (*in_buf) >> 30;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 3)) << (5 - 3);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 3) % (1u32 << 5);
- out = out.offset(1);
- *out = ((*in_buf) >> 8) % (1u32 << 5);
- out = out.offset(1);
- *out = ((*in_buf) >> 13) % (1u32 << 5);
- out = out.offset(1);
- *out = ((*in_buf) >> 18) % (1u32 << 5);
- out = out.offset(1);
- *out = ((*in_buf) >> 23) % (1u32 << 5);
- out = out.offset(1);
- *out = ((*in_buf) >> 28) % (1u32 << 5);
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 1)) << (5 - 1);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 1) % (1u32 << 5);
- out = out.offset(1);
- *out = ((*in_buf) >> 6) % (1u32 << 5);
- out = out.offset(1);
- *out = ((*in_buf) >> 11) % (1u32 << 5);
- out = out.offset(1);
- *out = ((*in_buf) >> 16) % (1u32 << 5);
- out = out.offset(1);
- *out = ((*in_buf) >> 21) % (1u32 << 5);
- out = out.offset(1);
- *out = ((*in_buf) >> 26) % (1u32 << 5);
- out = out.offset(1);
- *out = (*in_buf) >> 31;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 4)) << (5 - 4);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 4) % (1u32 << 5);
- out = out.offset(1);
- *out = ((*in_buf) >> 9) % (1u32 << 5);
- out = out.offset(1);
- *out = ((*in_buf) >> 14) % (1u32 << 5);
- out = out.offset(1);
- *out = ((*in_buf) >> 19) % (1u32 << 5);
- out = out.offset(1);
- *out = ((*in_buf) >> 24) % (1u32 << 5);
- out = out.offset(1);
- *out = (*in_buf) >> 29;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 2)) << (5 - 2);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 2) % (1u32 << 5);
- out = out.offset(1);
- *out = ((*in_buf) >> 7) % (1u32 << 5);
- out = out.offset(1);
- *out = ((*in_buf) >> 12) % (1u32 << 5);
- out = out.offset(1);
- *out = ((*in_buf) >> 17) % (1u32 << 5);
- out = out.offset(1);
- *out = ((*in_buf) >> 22) % (1u32 << 5);
- out = out.offset(1);
- *out = (*in_buf) >> 27;
-
- in_buf.offset(1)
-}
-
-unsafe fn unpack6_32(mut in_buf: *const u32, mut out: *mut u32) -> *const u32 {
- *out = (*in_buf) % (1u32 << 6);
- out = out.offset(1);
- *out = ((*in_buf) >> 6) % (1u32 << 6);
- out = out.offset(1);
- *out = ((*in_buf) >> 12) % (1u32 << 6);
- out = out.offset(1);
- *out = ((*in_buf) >> 18) % (1u32 << 6);
- out = out.offset(1);
- *out = ((*in_buf) >> 24) % (1u32 << 6);
- out = out.offset(1);
- *out = (*in_buf) >> 30;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 4)) << (6 - 4);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 4) % (1u32 << 6);
- out = out.offset(1);
- *out = ((*in_buf) >> 10) % (1u32 << 6);
- out = out.offset(1);
- *out = ((*in_buf) >> 16) % (1u32 << 6);
- out = out.offset(1);
- *out = ((*in_buf) >> 22) % (1u32 << 6);
- out = out.offset(1);
- *out = (*in_buf) >> 28;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 2)) << (6 - 2);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 2) % (1u32 << 6);
- out = out.offset(1);
- *out = ((*in_buf) >> 8) % (1u32 << 6);
- out = out.offset(1);
- *out = ((*in_buf) >> 14) % (1u32 << 6);
- out = out.offset(1);
- *out = ((*in_buf) >> 20) % (1u32 << 6);
- out = out.offset(1);
- *out = (*in_buf) >> 26;
- in_buf = in_buf.offset(1);
- out = out.offset(1);
-
- *out = (*in_buf) % (1u32 << 6);
- out = out.offset(1);
- *out = ((*in_buf) >> 6) % (1u32 << 6);
- out = out.offset(1);
- *out = ((*in_buf) >> 12) % (1u32 << 6);
- out = out.offset(1);
- *out = ((*in_buf) >> 18) % (1u32 << 6);
- out = out.offset(1);
- *out = ((*in_buf) >> 24) % (1u32 << 6);
- out = out.offset(1);
- *out = (*in_buf) >> 30;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 4)) << (6 - 4);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 4) % (1u32 << 6);
- out = out.offset(1);
- *out = ((*in_buf) >> 10) % (1u32 << 6);
- out = out.offset(1);
- *out = ((*in_buf) >> 16) % (1u32 << 6);
- out = out.offset(1);
- *out = ((*in_buf) >> 22) % (1u32 << 6);
- out = out.offset(1);
- *out = (*in_buf) >> 28;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 2)) << (6 - 2);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 2) % (1u32 << 6);
- out = out.offset(1);
- *out = ((*in_buf) >> 8) % (1u32 << 6);
- out = out.offset(1);
- *out = ((*in_buf) >> 14) % (1u32 << 6);
- out = out.offset(1);
- *out = ((*in_buf) >> 20) % (1u32 << 6);
- out = out.offset(1);
- *out = (*in_buf) >> 26;
-
- in_buf.offset(1)
-}
-
-unsafe fn unpack7_32(mut in_buf: *const u32, mut out: *mut u32) -> *const u32 {
- *out = (*in_buf) % (1u32 << 7);
- out = out.offset(1);
- *out = ((*in_buf) >> 7) % (1u32 << 7);
- out = out.offset(1);
- *out = ((*in_buf) >> 14) % (1u32 << 7);
- out = out.offset(1);
- *out = ((*in_buf) >> 21) % (1u32 << 7);
- out = out.offset(1);
- *out = (*in_buf) >> 28;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 3)) << (7 - 3);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 3) % (1u32 << 7);
- out = out.offset(1);
- *out = ((*in_buf) >> 10) % (1u32 << 7);
- out = out.offset(1);
- *out = ((*in_buf) >> 17) % (1u32 << 7);
- out = out.offset(1);
- *out = ((*in_buf) >> 24) % (1u32 << 7);
- out = out.offset(1);
- *out = (*in_buf) >> 31;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 6)) << (7 - 6);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 6) % (1u32 << 7);
- out = out.offset(1);
- *out = ((*in_buf) >> 13) % (1u32 << 7);
- out = out.offset(1);
- *out = ((*in_buf) >> 20) % (1u32 << 7);
- out = out.offset(1);
- *out = (*in_buf) >> 27;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 2)) << (7 - 2);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 2) % (1u32 << 7);
- out = out.offset(1);
- *out = ((*in_buf) >> 9) % (1u32 << 7);
- out = out.offset(1);
- *out = ((*in_buf) >> 16) % (1u32 << 7);
- out = out.offset(1);
- *out = ((*in_buf) >> 23) % (1u32 << 7);
- out = out.offset(1);
- *out = (*in_buf) >> 30;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 5)) << (7 - 5);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 5) % (1u32 << 7);
- out = out.offset(1);
- *out = ((*in_buf) >> 12) % (1u32 << 7);
- out = out.offset(1);
- *out = ((*in_buf) >> 19) % (1u32 << 7);
- out = out.offset(1);
- *out = (*in_buf) >> 26;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 1)) << (7 - 1);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 1) % (1u32 << 7);
- out = out.offset(1);
- *out = ((*in_buf) >> 8) % (1u32 << 7);
- out = out.offset(1);
- *out = ((*in_buf) >> 15) % (1u32 << 7);
- out = out.offset(1);
- *out = ((*in_buf) >> 22) % (1u32 << 7);
- out = out.offset(1);
- *out = (*in_buf) >> 29;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 4)) << (7 - 4);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 4) % (1u32 << 7);
- out = out.offset(1);
- *out = ((*in_buf) >> 11) % (1u32 << 7);
- out = out.offset(1);
- *out = ((*in_buf) >> 18) % (1u32 << 7);
- out = out.offset(1);
- *out = (*in_buf) >> 25;
-
- in_buf.offset(1)
-}
-
-unsafe fn unpack8_32(mut in_buf: *const u32, mut out: *mut u32) -> *const u32 {
- *out = (*in_buf) % (1u32 << 8);
- out = out.offset(1);
- *out = ((*in_buf) >> 8) % (1u32 << 8);
- out = out.offset(1);
- *out = ((*in_buf) >> 16) % (1u32 << 8);
- out = out.offset(1);
- *out = (*in_buf) >> 24;
- out = out.offset(1);
- in_buf = in_buf.offset(1);
-
- *out = (*in_buf) % (1u32 << 8);
- out = out.offset(1);
- *out = ((*in_buf) >> 8) % (1u32 << 8);
- out = out.offset(1);
- *out = ((*in_buf) >> 16) % (1u32 << 8);
- out = out.offset(1);
- *out = (*in_buf) >> 24;
- out = out.offset(1);
- in_buf = in_buf.offset(1);
-
- *out = (*in_buf) % (1u32 << 8);
- out = out.offset(1);
- *out = ((*in_buf) >> 8) % (1u32 << 8);
- out = out.offset(1);
- *out = ((*in_buf) >> 16) % (1u32 << 8);
- out = out.offset(1);
- *out = (*in_buf) >> 24;
- out = out.offset(1);
- in_buf = in_buf.offset(1);
-
- *out = (*in_buf) % (1u32 << 8);
- out = out.offset(1);
- *out = ((*in_buf) >> 8) % (1u32 << 8);
- out = out.offset(1);
- *out = ((*in_buf) >> 16) % (1u32 << 8);
- out = out.offset(1);
- *out = (*in_buf) >> 24;
- out = out.offset(1);
- in_buf = in_buf.offset(1);
-
- *out = (*in_buf) % (1u32 << 8);
- out = out.offset(1);
- *out = ((*in_buf) >> 8) % (1u32 << 8);
- out = out.offset(1);
- *out = ((*in_buf) >> 16) % (1u32 << 8);
- out = out.offset(1);
- *out = (*in_buf) >> 24;
- out = out.offset(1);
- in_buf = in_buf.offset(1);
-
- *out = (*in_buf) % (1u32 << 8);
- out = out.offset(1);
- *out = ((*in_buf) >> 8) % (1u32 << 8);
- out = out.offset(1);
- *out = ((*in_buf) >> 16) % (1u32 << 8);
- out = out.offset(1);
- *out = (*in_buf) >> 24;
- out = out.offset(1);
- in_buf = in_buf.offset(1);
-
- *out = (*in_buf) % (1u32 << 8);
- out = out.offset(1);
- *out = ((*in_buf) >> 8) % (1u32 << 8);
- out = out.offset(1);
- *out = ((*in_buf) >> 16) % (1u32 << 8);
- out = out.offset(1);
- *out = (*in_buf) >> 24;
- out = out.offset(1);
- in_buf = in_buf.offset(1);
-
- *out = (*in_buf) % (1u32 << 8);
- out = out.offset(1);
- *out = ((*in_buf) >> 8) % (1u32 << 8);
- out = out.offset(1);
- *out = ((*in_buf) >> 16) % (1u32 << 8);
- out = out.offset(1);
- *out = (*in_buf) >> 24;
-
- in_buf.offset(1)
-}
-
-unsafe fn unpack9_32(mut in_buf: *const u32, mut out: *mut u32) -> *const u32 {
- *out = (*in_buf) % (1u32 << 9);
- out = out.offset(1);
- *out = ((*in_buf) >> 9) % (1u32 << 9);
- out = out.offset(1);
- *out = ((*in_buf) >> 18) % (1u32 << 9);
- out = out.offset(1);
- *out = (*in_buf) >> 27;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 4)) << (9 - 4);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 4) % (1u32 << 9);
- out = out.offset(1);
- *out = ((*in_buf) >> 13) % (1u32 << 9);
- out = out.offset(1);
- *out = ((*in_buf) >> 22) % (1u32 << 9);
- out = out.offset(1);
- *out = (*in_buf) >> 31;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 8)) << (9 - 8);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 8) % (1u32 << 9);
- out = out.offset(1);
- *out = ((*in_buf) >> 17) % (1u32 << 9);
- out = out.offset(1);
- *out = (*in_buf) >> 26;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 3)) << (9 - 3);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 3) % (1u32 << 9);
- out = out.offset(1);
- *out = ((*in_buf) >> 12) % (1u32 << 9);
- out = out.offset(1);
- *out = ((*in_buf) >> 21) % (1u32 << 9);
- out = out.offset(1);
- *out = (*in_buf) >> 30;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 7)) << (9 - 7);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 7) % (1u32 << 9);
- out = out.offset(1);
- *out = ((*in_buf) >> 16) % (1u32 << 9);
- out = out.offset(1);
- *out = (*in_buf) >> 25;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 2)) << (9 - 2);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 2) % (1u32 << 9);
- out = out.offset(1);
- *out = ((*in_buf) >> 11) % (1u32 << 9);
- out = out.offset(1);
- *out = ((*in_buf) >> 20) % (1u32 << 9);
- out = out.offset(1);
- *out = (*in_buf) >> 29;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 6)) << (9 - 6);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 6) % (1u32 << 9);
- out = out.offset(1);
- *out = ((*in_buf) >> 15) % (1u32 << 9);
- out = out.offset(1);
- *out = (*in_buf) >> 24;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 1)) << (9 - 1);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 1) % (1u32 << 9);
- out = out.offset(1);
- *out = ((*in_buf) >> 10) % (1u32 << 9);
- out = out.offset(1);
- *out = ((*in_buf) >> 19) % (1u32 << 9);
- out = out.offset(1);
- *out = (*in_buf) >> 28;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 5)) << (9 - 5);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 5) % (1u32 << 9);
- out = out.offset(1);
- *out = ((*in_buf) >> 14) % (1u32 << 9);
- out = out.offset(1);
- *out = (*in_buf) >> 23;
-
- in_buf.offset(1)
-}
-
-unsafe fn unpack10_32(mut in_buf: *const u32, mut out: *mut u32) -> *const u32 {
- *out = (*in_buf) % (1u32 << 10);
- out = out.offset(1);
- *out = ((*in_buf) >> 10) % (1u32 << 10);
- out = out.offset(1);
- *out = ((*in_buf) >> 20) % (1u32 << 10);
- out = out.offset(1);
- *out = (*in_buf) >> 30;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 8)) << (10 - 8);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 8) % (1u32 << 10);
- out = out.offset(1);
- *out = ((*in_buf) >> 18) % (1u32 << 10);
- out = out.offset(1);
- *out = (*in_buf) >> 28;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 6)) << (10 - 6);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 6) % (1u32 << 10);
- out = out.offset(1);
- *out = ((*in_buf) >> 16) % (1u32 << 10);
- out = out.offset(1);
- *out = (*in_buf) >> 26;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 4)) << (10 - 4);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 4) % (1u32 << 10);
- out = out.offset(1);
- *out = ((*in_buf) >> 14) % (1u32 << 10);
- out = out.offset(1);
- *out = (*in_buf) >> 24;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 2)) << (10 - 2);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 2) % (1u32 << 10);
- out = out.offset(1);
- *out = ((*in_buf) >> 12) % (1u32 << 10);
- out = out.offset(1);
- *out = (*in_buf) >> 22;
- in_buf = in_buf.offset(1);
- out = out.offset(1);
-
- *out = (*in_buf) % (1u32 << 10);
- out = out.offset(1);
- *out = ((*in_buf) >> 10) % (1u32 << 10);
- out = out.offset(1);
- *out = ((*in_buf) >> 20) % (1u32 << 10);
- out = out.offset(1);
- *out = (*in_buf) >> 30;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 8)) << (10 - 8);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 8) % (1u32 << 10);
- out = out.offset(1);
- *out = ((*in_buf) >> 18) % (1u32 << 10);
- out = out.offset(1);
- *out = (*in_buf) >> 28;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 6)) << (10 - 6);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 6) % (1u32 << 10);
- out = out.offset(1);
- *out = ((*in_buf) >> 16) % (1u32 << 10);
- out = out.offset(1);
- *out = (*in_buf) >> 26;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 4)) << (10 - 4);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 4) % (1u32 << 10);
- out = out.offset(1);
- *out = ((*in_buf) >> 14) % (1u32 << 10);
- out = out.offset(1);
- *out = (*in_buf) >> 24;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 2)) << (10 - 2);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 2) % (1u32 << 10);
- out = out.offset(1);
- *out = ((*in_buf) >> 12) % (1u32 << 10);
- out = out.offset(1);
- *out = (*in_buf) >> 22;
-
- in_buf.offset(1)
-}
-
-unsafe fn unpack11_32(mut in_buf: *const u32, mut out: *mut u32) -> *const u32 {
- *out = (*in_buf) % (1u32 << 11);
- out = out.offset(1);
- *out = ((*in_buf) >> 11) % (1u32 << 11);
- out = out.offset(1);
- *out = (*in_buf) >> 22;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 1)) << (11 - 1);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 1) % (1u32 << 11);
- out = out.offset(1);
- *out = ((*in_buf) >> 12) % (1u32 << 11);
- out = out.offset(1);
- *out = (*in_buf) >> 23;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 2)) << (11 - 2);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 2) % (1u32 << 11);
- out = out.offset(1);
- *out = ((*in_buf) >> 13) % (1u32 << 11);
- out = out.offset(1);
- *out = (*in_buf) >> 24;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 3)) << (11 - 3);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 3) % (1u32 << 11);
- out = out.offset(1);
- *out = ((*in_buf) >> 14) % (1u32 << 11);
- out = out.offset(1);
- *out = (*in_buf) >> 25;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 4)) << (11 - 4);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 4) % (1u32 << 11);
- out = out.offset(1);
- *out = ((*in_buf) >> 15) % (1u32 << 11);
- out = out.offset(1);
- *out = (*in_buf) >> 26;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 5)) << (11 - 5);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 5) % (1u32 << 11);
- out = out.offset(1);
- *out = ((*in_buf) >> 16) % (1u32 << 11);
- out = out.offset(1);
- *out = (*in_buf) >> 27;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 6)) << (11 - 6);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 6) % (1u32 << 11);
- out = out.offset(1);
- *out = ((*in_buf) >> 17) % (1u32 << 11);
- out = out.offset(1);
- *out = (*in_buf) >> 28;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 7)) << (11 - 7);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 7) % (1u32 << 11);
- out = out.offset(1);
- *out = ((*in_buf) >> 18) % (1u32 << 11);
- out = out.offset(1);
- *out = (*in_buf) >> 29;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 8)) << (11 - 8);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 8) % (1u32 << 11);
- out = out.offset(1);
- *out = ((*in_buf) >> 19) % (1u32 << 11);
- out = out.offset(1);
- *out = (*in_buf) >> 30;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 9)) << (11 - 9);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 9) % (1u32 << 11);
- out = out.offset(1);
- *out = ((*in_buf) >> 20) % (1u32 << 11);
- out = out.offset(1);
- *out = (*in_buf) >> 31;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 10)) << (11 - 10);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 10) % (1u32 << 11);
- out = out.offset(1);
- *out = (*in_buf) >> 21;
-
- in_buf.offset(1)
-}
-
-unsafe fn unpack12_32(mut in_buf: *const u32, mut out: *mut u32) -> *const u32 {
- *out = (*in_buf) % (1u32 << 12);
- out = out.offset(1);
- *out = ((*in_buf) >> 12) % (1u32 << 12);
- out = out.offset(1);
- *out = (*in_buf) >> 24;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 4)) << (12 - 4);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 4) % (1u32 << 12);
- out = out.offset(1);
- *out = ((*in_buf) >> 16) % (1u32 << 12);
- out = out.offset(1);
- *out = (*in_buf) >> 28;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 8)) << (12 - 8);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 8) % (1u32 << 12);
- out = out.offset(1);
- *out = (*in_buf) >> 20;
- in_buf = in_buf.offset(1);
- out = out.offset(1);
-
- *out = (*in_buf) % (1u32 << 12);
- out = out.offset(1);
- *out = ((*in_buf) >> 12) % (1u32 << 12);
- out = out.offset(1);
- *out = (*in_buf) >> 24;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 4)) << (12 - 4);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 4) % (1u32 << 12);
- out = out.offset(1);
- *out = ((*in_buf) >> 16) % (1u32 << 12);
- out = out.offset(1);
- *out = (*in_buf) >> 28;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 8)) << (12 - 8);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 8) % (1u32 << 12);
- out = out.offset(1);
- *out = (*in_buf) >> 20;
- in_buf = in_buf.offset(1);
- out = out.offset(1);
-
- *out = (*in_buf) % (1u32 << 12);
- out = out.offset(1);
- *out = ((*in_buf) >> 12) % (1u32 << 12);
- out = out.offset(1);
- *out = (*in_buf) >> 24;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 4)) << (12 - 4);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 4) % (1u32 << 12);
- out = out.offset(1);
- *out = ((*in_buf) >> 16) % (1u32 << 12);
- out = out.offset(1);
- *out = (*in_buf) >> 28;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 8)) << (12 - 8);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 8) % (1u32 << 12);
- out = out.offset(1);
- *out = (*in_buf) >> 20;
- in_buf = in_buf.offset(1);
- out = out.offset(1);
-
- *out = (*in_buf) % (1u32 << 12);
- out = out.offset(1);
- *out = ((*in_buf) >> 12) % (1u32 << 12);
- out = out.offset(1);
- *out = (*in_buf) >> 24;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 4)) << (12 - 4);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 4) % (1u32 << 12);
- out = out.offset(1);
- *out = ((*in_buf) >> 16) % (1u32 << 12);
- out = out.offset(1);
- *out = (*in_buf) >> 28;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 8)) << (12 - 8);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 8) % (1u32 << 12);
- out = out.offset(1);
- *out = (*in_buf) >> 20;
-
- in_buf.offset(1)
-}
-
-unsafe fn unpack13_32(mut in_buf: *const u32, mut out: *mut u32) -> *const u32 {
- *out = (*in_buf) % (1u32 << 13);
- out = out.offset(1);
- *out = ((*in_buf) >> 13) % (1u32 << 13);
- out = out.offset(1);
- *out = (*in_buf) >> 26;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 7)) << (13 - 7);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 7) % (1u32 << 13);
- out = out.offset(1);
- *out = (*in_buf) >> 20;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 1)) << (13 - 1);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 1) % (1u32 << 13);
- out = out.offset(1);
- *out = ((*in_buf) >> 14) % (1u32 << 13);
- out = out.offset(1);
- *out = (*in_buf) >> 27;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 8)) << (13 - 8);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 8) % (1u32 << 13);
- out = out.offset(1);
- *out = (*in_buf) >> 21;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 2)) << (13 - 2);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 2) % (1u32 << 13);
- out = out.offset(1);
- *out = ((*in_buf) >> 15) % (1u32 << 13);
- out = out.offset(1);
- *out = (*in_buf) >> 28;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 9)) << (13 - 9);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 9) % (1u32 << 13);
- out = out.offset(1);
- *out = (*in_buf) >> 22;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 3)) << (13 - 3);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 3) % (1u32 << 13);
- out = out.offset(1);
- *out = ((*in_buf) >> 16) % (1u32 << 13);
- out = out.offset(1);
- *out = (*in_buf) >> 29;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 10)) << (13 - 10);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 10) % (1u32 << 13);
- out = out.offset(1);
- *out = (*in_buf) >> 23;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 4)) << (13 - 4);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 4) % (1u32 << 13);
- out = out.offset(1);
- *out = ((*in_buf) >> 17) % (1u32 << 13);
- out = out.offset(1);
- *out = (*in_buf) >> 30;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 11)) << (13 - 11);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 11) % (1u32 << 13);
- out = out.offset(1);
- *out = (*in_buf) >> 24;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 5)) << (13 - 5);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 5) % (1u32 << 13);
- out = out.offset(1);
- *out = ((*in_buf) >> 18) % (1u32 << 13);
- out = out.offset(1);
- *out = (*in_buf) >> 31;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 12)) << (13 - 12);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 12) % (1u32 << 13);
- out = out.offset(1);
- *out = (*in_buf) >> 25;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 6)) << (13 - 6);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 6) % (1u32 << 13);
- out = out.offset(1);
- *out = (*in_buf) >> 19;
-
- in_buf.offset(1)
-}
-
-unsafe fn unpack14_32(mut in_buf: *const u32, mut out: *mut u32) -> *const u32 {
- *out = (*in_buf) % (1u32 << 14);
- out = out.offset(1);
- *out = ((*in_buf) >> 14) % (1u32 << 14);
- out = out.offset(1);
- *out = (*in_buf) >> 28;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 10)) << (14 - 10);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 10) % (1u32 << 14);
- out = out.offset(1);
- *out = (*in_buf) >> 24;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 6)) << (14 - 6);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 6) % (1u32 << 14);
- out = out.offset(1);
- *out = (*in_buf) >> 20;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 2)) << (14 - 2);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 2) % (1u32 << 14);
- out = out.offset(1);
- *out = ((*in_buf) >> 16) % (1u32 << 14);
- out = out.offset(1);
- *out = (*in_buf) >> 30;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 12)) << (14 - 12);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 12) % (1u32 << 14);
- out = out.offset(1);
- *out = (*in_buf) >> 26;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 8)) << (14 - 8);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 8) % (1u32 << 14);
- out = out.offset(1);
- *out = (*in_buf) >> 22;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 4)) << (14 - 4);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 4) % (1u32 << 14);
- out = out.offset(1);
- *out = (*in_buf) >> 18;
- in_buf = in_buf.offset(1);
- out = out.offset(1);
-
- *out = (*in_buf) % (1u32 << 14);
- out = out.offset(1);
- *out = ((*in_buf) >> 14) % (1u32 << 14);
- out = out.offset(1);
- *out = (*in_buf) >> 28;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 10)) << (14 - 10);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 10) % (1u32 << 14);
- out = out.offset(1);
- *out = (*in_buf) >> 24;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 6)) << (14 - 6);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 6) % (1u32 << 14);
- out = out.offset(1);
- *out = (*in_buf) >> 20;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 2)) << (14 - 2);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 2) % (1u32 << 14);
- out = out.offset(1);
- *out = ((*in_buf) >> 16) % (1u32 << 14);
- out = out.offset(1);
- *out = (*in_buf) >> 30;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 12)) << (14 - 12);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 12) % (1u32 << 14);
- out = out.offset(1);
- *out = (*in_buf) >> 26;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 8)) << (14 - 8);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 8) % (1u32 << 14);
- out = out.offset(1);
- *out = (*in_buf) >> 22;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 4)) << (14 - 4);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 4) % (1u32 << 14);
- out = out.offset(1);
- *out = (*in_buf) >> 18;
-
- in_buf.offset(1)
-}
-
-unsafe fn unpack15_32(mut in_buf: *const u32, mut out: *mut u32) -> *const u32 {
- *out = (*in_buf) % (1u32 << 15);
- out = out.offset(1);
- *out = ((*in_buf) >> 15) % (1u32 << 15);
- out = out.offset(1);
- *out = (*in_buf) >> 30;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 13)) << (15 - 13);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 13) % (1u32 << 15);
- out = out.offset(1);
- *out = (*in_buf) >> 28;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 11)) << (15 - 11);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 11) % (1u32 << 15);
- out = out.offset(1);
- *out = (*in_buf) >> 26;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 9)) << (15 - 9);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 9) % (1u32 << 15);
- out = out.offset(1);
- *out = (*in_buf) >> 24;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 7)) << (15 - 7);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 7) % (1u32 << 15);
- out = out.offset(1);
- *out = (*in_buf) >> 22;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 5)) << (15 - 5);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 5) % (1u32 << 15);
- out = out.offset(1);
- *out = (*in_buf) >> 20;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 3)) << (15 - 3);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 3) % (1u32 << 15);
- out = out.offset(1);
- *out = (*in_buf) >> 18;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 1)) << (15 - 1);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 1) % (1u32 << 15);
- out = out.offset(1);
- *out = ((*in_buf) >> 16) % (1u32 << 15);
- out = out.offset(1);
- *out = (*in_buf) >> 31;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 14)) << (15 - 14);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 14) % (1u32 << 15);
- out = out.offset(1);
- *out = (*in_buf) >> 29;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 12)) << (15 - 12);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 12) % (1u32 << 15);
- out = out.offset(1);
- *out = (*in_buf) >> 27;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 10)) << (15 - 10);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 10) % (1u32 << 15);
- out = out.offset(1);
- *out = (*in_buf) >> 25;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 8)) << (15 - 8);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 8) % (1u32 << 15);
- out = out.offset(1);
- *out = (*in_buf) >> 23;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 6)) << (15 - 6);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 6) % (1u32 << 15);
- out = out.offset(1);
- *out = (*in_buf) >> 21;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 4)) << (15 - 4);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 4) % (1u32 << 15);
- out = out.offset(1);
- *out = (*in_buf) >> 19;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 2)) << (15 - 2);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 2) % (1u32 << 15);
- out = out.offset(1);
- *out = (*in_buf) >> 17;
-
- in_buf.offset(1)
-}
-
-unsafe fn unpack16_32(mut in_buf: *const u32, mut out: *mut u32) -> *const u32 {
- *out = (*in_buf) % (1u32 << 16);
- out = out.offset(1);
- *out = (*in_buf) >> 16;
- out = out.offset(1);
- in_buf = in_buf.offset(1);
-
- *out = (*in_buf) % (1u32 << 16);
- out = out.offset(1);
- *out = (*in_buf) >> 16;
- out = out.offset(1);
- in_buf = in_buf.offset(1);
-
- *out = (*in_buf) % (1u32 << 16);
- out = out.offset(1);
- *out = (*in_buf) >> 16;
- out = out.offset(1);
- in_buf = in_buf.offset(1);
-
- *out = (*in_buf) % (1u32 << 16);
- out = out.offset(1);
- *out = (*in_buf) >> 16;
- out = out.offset(1);
- in_buf = in_buf.offset(1);
-
- *out = (*in_buf) % (1u32 << 16);
- out = out.offset(1);
- *out = (*in_buf) >> 16;
- out = out.offset(1);
- in_buf = in_buf.offset(1);
-
- *out = (*in_buf) % (1u32 << 16);
- out = out.offset(1);
- *out = (*in_buf) >> 16;
- out = out.offset(1);
- in_buf = in_buf.offset(1);
-
- *out = (*in_buf) % (1u32 << 16);
- out = out.offset(1);
- *out = (*in_buf) >> 16;
- out = out.offset(1);
- in_buf = in_buf.offset(1);
-
- *out = (*in_buf) % (1u32 << 16);
- out = out.offset(1);
- *out = (*in_buf) >> 16;
- out = out.offset(1);
- in_buf = in_buf.offset(1);
-
- *out = (*in_buf) % (1u32 << 16);
- out = out.offset(1);
- *out = (*in_buf) >> 16;
- out = out.offset(1);
- in_buf = in_buf.offset(1);
-
- *out = (*in_buf) % (1u32 << 16);
- out = out.offset(1);
- *out = (*in_buf) >> 16;
- out = out.offset(1);
- in_buf = in_buf.offset(1);
-
- *out = (*in_buf) % (1u32 << 16);
- out = out.offset(1);
- *out = (*in_buf) >> 16;
- out = out.offset(1);
- in_buf = in_buf.offset(1);
-
- *out = (*in_buf) % (1u32 << 16);
- out = out.offset(1);
- *out = (*in_buf) >> 16;
- out = out.offset(1);
- in_buf = in_buf.offset(1);
-
- *out = (*in_buf) % (1u32 << 16);
- out = out.offset(1);
- *out = (*in_buf) >> 16;
- out = out.offset(1);
- in_buf = in_buf.offset(1);
-
- *out = (*in_buf) % (1u32 << 16);
- out = out.offset(1);
- *out = (*in_buf) >> 16;
- out = out.offset(1);
- in_buf = in_buf.offset(1);
-
- *out = (*in_buf) % (1u32 << 16);
- out = out.offset(1);
- *out = (*in_buf) >> 16;
- out = out.offset(1);
- in_buf = in_buf.offset(1);
-
- *out = (*in_buf) % (1u32 << 16);
- out = out.offset(1);
- *out = (*in_buf) >> 16;
-
- in_buf.offset(1)
-}
-
-unsafe fn unpack17_32(mut in_buf: *const u32, mut out: *mut u32) -> *const u32 {
- *out = (*in_buf) % (1u32 << 17);
- out = out.offset(1);
- *out = (*in_buf) >> 17;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 2)) << (17 - 2);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 2) % (1u32 << 17);
- out = out.offset(1);
- *out = (*in_buf) >> 19;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 4)) << (17 - 4);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 4) % (1u32 << 17);
- out = out.offset(1);
- *out = (*in_buf) >> 21;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 6)) << (17 - 6);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 6) % (1u32 << 17);
- out = out.offset(1);
- *out = (*in_buf) >> 23;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 8)) << (17 - 8);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 8) % (1u32 << 17);
- out = out.offset(1);
- *out = (*in_buf) >> 25;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 10)) << (17 - 10);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 10) % (1u32 << 17);
- out = out.offset(1);
- *out = (*in_buf) >> 27;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 12)) << (17 - 12);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 12) % (1u32 << 17);
- out = out.offset(1);
- *out = (*in_buf) >> 29;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 14)) << (17 - 14);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 14) % (1u32 << 17);
- out = out.offset(1);
- *out = (*in_buf) >> 31;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 16)) << (17 - 16);
- out = out.offset(1);
-
- *out = (*in_buf) >> 16;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 1)) << (17 - 1);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 1) % (1u32 << 17);
- out = out.offset(1);
- *out = (*in_buf) >> 18;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 3)) << (17 - 3);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 3) % (1u32 << 17);
- out = out.offset(1);
- *out = (*in_buf) >> 20;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 5)) << (17 - 5);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 5) % (1u32 << 17);
- out = out.offset(1);
- *out = (*in_buf) >> 22;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 7)) << (17 - 7);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 7) % (1u32 << 17);
- out = out.offset(1);
- *out = (*in_buf) >> 24;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 9)) << (17 - 9);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 9) % (1u32 << 17);
- out = out.offset(1);
- *out = (*in_buf) >> 26;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 11)) << (17 - 11);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 11) % (1u32 << 17);
- out = out.offset(1);
- *out = (*in_buf) >> 28;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 13)) << (17 - 13);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 13) % (1u32 << 17);
- out = out.offset(1);
- *out = (*in_buf) >> 30;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 15)) << (17 - 15);
- out = out.offset(1);
-
- *out = (*in_buf) >> 15;
-
- in_buf.offset(1)
-}
-
-unsafe fn unpack18_32(mut in_buf: *const u32, mut out: *mut u32) -> *const u32 {
- *out = (*in_buf) % (1u32 << 18);
- out = out.offset(1);
- *out = (*in_buf) >> 18;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 4)) << (18 - 4);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 4) % (1u32 << 18);
- out = out.offset(1);
- *out = (*in_buf) >> 22;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 8)) << (18 - 8);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 8) % (1u32 << 18);
- out = out.offset(1);
- *out = (*in_buf) >> 26;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 12)) << (18 - 12);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 12) % (1u32 << 18);
- out = out.offset(1);
- *out = (*in_buf) >> 30;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 16)) << (18 - 16);
- out = out.offset(1);
-
- *out = (*in_buf) >> 16;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 2)) << (18 - 2);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 2) % (1u32 << 18);
- out = out.offset(1);
- *out = (*in_buf) >> 20;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 6)) << (18 - 6);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 6) % (1u32 << 18);
- out = out.offset(1);
- *out = (*in_buf) >> 24;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 10)) << (18 - 10);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 10) % (1u32 << 18);
- out = out.offset(1);
- *out = (*in_buf) >> 28;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 14)) << (18 - 14);
- out = out.offset(1);
-
- *out = (*in_buf) >> 14;
- in_buf = in_buf.offset(1);
- out = out.offset(1);
-
- *out = (*in_buf) % (1u32 << 18);
- out = out.offset(1);
- *out = (*in_buf) >> 18;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 4)) << (18 - 4);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 4) % (1u32 << 18);
- out = out.offset(1);
- *out = (*in_buf) >> 22;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 8)) << (18 - 8);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 8) % (1u32 << 18);
- out = out.offset(1);
- *out = (*in_buf) >> 26;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 12)) << (18 - 12);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 12) % (1u32 << 18);
- out = out.offset(1);
- *out = (*in_buf) >> 30;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 16)) << (18 - 16);
- out = out.offset(1);
-
- *out = (*in_buf) >> 16;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 2)) << (18 - 2);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 2) % (1u32 << 18);
- out = out.offset(1);
- *out = (*in_buf) >> 20;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 6)) << (18 - 6);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 6) % (1u32 << 18);
- out = out.offset(1);
- *out = (*in_buf) >> 24;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 10)) << (18 - 10);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 10) % (1u32 << 18);
- out = out.offset(1);
- *out = (*in_buf) >> 28;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 14)) << (18 - 14);
- out = out.offset(1);
-
- *out = (*in_buf) >> 14;
-
- in_buf.offset(1)
-}
-
-unsafe fn unpack19_32(mut in_buf: *const u32, mut out: *mut u32) -> *const u32 {
- *out = (*in_buf) % (1u32 << 19);
- out = out.offset(1);
- *out = (*in_buf) >> 19;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 6)) << (19 - 6);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 6) % (1u32 << 19);
- out = out.offset(1);
- *out = (*in_buf) >> 25;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 12)) << (19 - 12);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 12) % (1u32 << 19);
- out = out.offset(1);
- *out = (*in_buf) >> 31;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 18)) << (19 - 18);
- out = out.offset(1);
-
- *out = (*in_buf) >> 18;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 5)) << (19 - 5);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 5) % (1u32 << 19);
- out = out.offset(1);
- *out = (*in_buf) >> 24;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 11)) << (19 - 11);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 11) % (1u32 << 19);
- out = out.offset(1);
- *out = (*in_buf) >> 30;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 17)) << (19 - 17);
- out = out.offset(1);
-
- *out = (*in_buf) >> 17;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 4)) << (19 - 4);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 4) % (1u32 << 19);
- out = out.offset(1);
- *out = (*in_buf) >> 23;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 10)) << (19 - 10);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 10) % (1u32 << 19);
- out = out.offset(1);
- *out = (*in_buf) >> 29;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 16)) << (19 - 16);
- out = out.offset(1);
-
- *out = (*in_buf) >> 16;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 3)) << (19 - 3);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 3) % (1u32 << 19);
- out = out.offset(1);
- *out = (*in_buf) >> 22;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 9)) << (19 - 9);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 9) % (1u32 << 19);
- out = out.offset(1);
- *out = (*in_buf) >> 28;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 15)) << (19 - 15);
- out = out.offset(1);
-
- *out = (*in_buf) >> 15;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 2)) << (19 - 2);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 2) % (1u32 << 19);
- out = out.offset(1);
- *out = (*in_buf) >> 21;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 8)) << (19 - 8);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 8) % (1u32 << 19);
- out = out.offset(1);
- *out = (*in_buf) >> 27;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 14)) << (19 - 14);
- out = out.offset(1);
-
- *out = (*in_buf) >> 14;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 1)) << (19 - 1);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 1) % (1u32 << 19);
- out = out.offset(1);
- *out = (*in_buf) >> 20;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 7)) << (19 - 7);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 7) % (1u32 << 19);
- out = out.offset(1);
- *out = (*in_buf) >> 26;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 13)) << (19 - 13);
- out = out.offset(1);
-
- *out = (*in_buf) >> 13;
-
- in_buf.offset(1)
-}
-
-unsafe fn unpack20_32(mut in_buf: *const u32, mut out: *mut u32) -> *const u32 {
- *out = (*in_buf) % (1u32 << 20);
- out = out.offset(1);
- *out = (*in_buf) >> 20;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 8)) << (20 - 8);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 8) % (1u32 << 20);
- out = out.offset(1);
- *out = (*in_buf) >> 28;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 16)) << (20 - 16);
- out = out.offset(1);
-
- *out = (*in_buf) >> 16;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 4)) << (20 - 4);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 4) % (1u32 << 20);
- out = out.offset(1);
- *out = (*in_buf) >> 24;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 12)) << (20 - 12);
- out = out.offset(1);
-
- *out = (*in_buf) >> 12;
- in_buf = in_buf.offset(1);
- out = out.offset(1);
-
- *out = (*in_buf) % (1u32 << 20);
- out = out.offset(1);
- *out = (*in_buf) >> 20;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 8)) << (20 - 8);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 8) % (1u32 << 20);
- out = out.offset(1);
- *out = (*in_buf) >> 28;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 16)) << (20 - 16);
- out = out.offset(1);
-
- *out = (*in_buf) >> 16;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 4)) << (20 - 4);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 4) % (1u32 << 20);
- out = out.offset(1);
- *out = (*in_buf) >> 24;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 12)) << (20 - 12);
- out = out.offset(1);
-
- *out = (*in_buf) >> 12;
- in_buf = in_buf.offset(1);
- out = out.offset(1);
-
- *out = (*in_buf) % (1u32 << 20);
- out = out.offset(1);
- *out = (*in_buf) >> 20;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 8)) << (20 - 8);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 8) % (1u32 << 20);
- out = out.offset(1);
- *out = (*in_buf) >> 28;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 16)) << (20 - 16);
- out = out.offset(1);
-
- *out = (*in_buf) >> 16;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 4)) << (20 - 4);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 4) % (1u32 << 20);
- out = out.offset(1);
- *out = (*in_buf) >> 24;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 12)) << (20 - 12);
- out = out.offset(1);
-
- *out = (*in_buf) >> 12;
- in_buf = in_buf.offset(1);
- out = out.offset(1);
-
- *out = (*in_buf) % (1u32 << 20);
- out = out.offset(1);
- *out = (*in_buf) >> 20;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 8)) << (20 - 8);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 8) % (1u32 << 20);
- out = out.offset(1);
- *out = (*in_buf) >> 28;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 16)) << (20 - 16);
- out = out.offset(1);
-
- *out = (*in_buf) >> 16;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 4)) << (20 - 4);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 4) % (1u32 << 20);
- out = out.offset(1);
- *out = (*in_buf) >> 24;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 12)) << (20 - 12);
- out = out.offset(1);
-
- *out = (*in_buf) >> 12;
-
- in_buf.offset(1)
-}
-
-unsafe fn unpack21_32(mut in_buf: *const u32, mut out: *mut u32) -> *const u32 {
- *out = (*in_buf) % (1u32 << 21);
- out = out.offset(1);
- *out = (*in_buf) >> 21;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 10)) << (21 - 10);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 10) % (1u32 << 21);
- out = out.offset(1);
- *out = (*in_buf) >> 31;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 20)) << (21 - 20);
- out = out.offset(1);
-
- *out = (*in_buf) >> 20;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 9)) << (21 - 9);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 9) % (1u32 << 21);
- out = out.offset(1);
- *out = (*in_buf) >> 30;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 19)) << (21 - 19);
- out = out.offset(1);
-
- *out = (*in_buf) >> 19;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 8)) << (21 - 8);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 8) % (1u32 << 21);
- out = out.offset(1);
- *out = (*in_buf) >> 29;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 18)) << (21 - 18);
- out = out.offset(1);
-
- *out = (*in_buf) >> 18;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 7)) << (21 - 7);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 7) % (1u32 << 21);
- out = out.offset(1);
- *out = (*in_buf) >> 28;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 17)) << (21 - 17);
- out = out.offset(1);
-
- *out = (*in_buf) >> 17;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 6)) << (21 - 6);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 6) % (1u32 << 21);
- out = out.offset(1);
- *out = (*in_buf) >> 27;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 16)) << (21 - 16);
- out = out.offset(1);
-
- *out = (*in_buf) >> 16;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 5)) << (21 - 5);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 5) % (1u32 << 21);
- out = out.offset(1);
- *out = (*in_buf) >> 26;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 15)) << (21 - 15);
- out = out.offset(1);
-
- *out = (*in_buf) >> 15;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 4)) << (21 - 4);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 4) % (1u32 << 21);
- out = out.offset(1);
- *out = (*in_buf) >> 25;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 14)) << (21 - 14);
- out = out.offset(1);
-
- *out = (*in_buf) >> 14;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 3)) << (21 - 3);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 3) % (1u32 << 21);
- out = out.offset(1);
- *out = (*in_buf) >> 24;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 13)) << (21 - 13);
- out = out.offset(1);
-
- *out = (*in_buf) >> 13;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 2)) << (21 - 2);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 2) % (1u32 << 21);
- out = out.offset(1);
- *out = (*in_buf) >> 23;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 12)) << (21 - 12);
- out = out.offset(1);
-
- *out = (*in_buf) >> 12;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 1)) << (21 - 1);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 1) % (1u32 << 21);
- out = out.offset(1);
- *out = (*in_buf) >> 22;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 11)) << (21 - 11);
- out = out.offset(1);
-
- *out = (*in_buf) >> 11;
-
- in_buf.offset(1)
-}
-
-unsafe fn unpack22_32(mut in_buf: *const u32, mut out: *mut u32) -> *const u32 {
- *out = (*in_buf) % (1u32 << 22);
- out = out.offset(1);
- *out = (*in_buf) >> 22;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 12)) << (22 - 12);
- out = out.offset(1);
-
- *out = (*in_buf) >> 12;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 2)) << (22 - 2);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 2) % (1u32 << 22);
- out = out.offset(1);
- *out = (*in_buf) >> 24;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 14)) << (22 - 14);
- out = out.offset(1);
-
- *out = (*in_buf) >> 14;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 4)) << (22 - 4);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 4) % (1u32 << 22);
- out = out.offset(1);
- *out = (*in_buf) >> 26;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 16)) << (22 - 16);
- out = out.offset(1);
-
- *out = (*in_buf) >> 16;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 6)) << (22 - 6);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 6) % (1u32 << 22);
- out = out.offset(1);
- *out = (*in_buf) >> 28;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 18)) << (22 - 18);
- out = out.offset(1);
-
- *out = (*in_buf) >> 18;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 8)) << (22 - 8);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 8) % (1u32 << 22);
- out = out.offset(1);
- *out = (*in_buf) >> 30;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 20)) << (22 - 20);
- out = out.offset(1);
-
- *out = (*in_buf) >> 20;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 10)) << (22 - 10);
- out = out.offset(1);
-
- *out = (*in_buf) >> 10;
- in_buf = in_buf.offset(1);
- out = out.offset(1);
-
- *out = (*in_buf) % (1u32 << 22);
- out = out.offset(1);
- *out = (*in_buf) >> 22;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 12)) << (22 - 12);
- out = out.offset(1);
-
- *out = (*in_buf) >> 12;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 2)) << (22 - 2);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 2) % (1u32 << 22);
- out = out.offset(1);
- *out = (*in_buf) >> 24;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 14)) << (22 - 14);
- out = out.offset(1);
-
- *out = (*in_buf) >> 14;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 4)) << (22 - 4);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 4) % (1u32 << 22);
- out = out.offset(1);
- *out = (*in_buf) >> 26;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 16)) << (22 - 16);
- out = out.offset(1);
-
- *out = (*in_buf) >> 16;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 6)) << (22 - 6);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 6) % (1u32 << 22);
- out = out.offset(1);
- *out = (*in_buf) >> 28;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 18)) << (22 - 18);
- out = out.offset(1);
-
- *out = (*in_buf) >> 18;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 8)) << (22 - 8);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 8) % (1u32 << 22);
- out = out.offset(1);
- *out = (*in_buf) >> 30;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 20)) << (22 - 20);
- out = out.offset(1);
-
- *out = (*in_buf) >> 20;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 10)) << (22 - 10);
- out = out.offset(1);
-
- *out = (*in_buf) >> 10;
-
- in_buf.offset(1)
-}
-
-unsafe fn unpack23_32(mut in_buf: *const u32, mut out: *mut u32) -> *const u32 {
- *out = (*in_buf) % (1u32 << 23);
- out = out.offset(1);
- *out = (*in_buf) >> 23;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 14)) << (23 - 14);
- out = out.offset(1);
-
- *out = (*in_buf) >> 14;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 5)) << (23 - 5);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 5) % (1u32 << 23);
- out = out.offset(1);
- *out = (*in_buf) >> 28;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 19)) << (23 - 19);
- out = out.offset(1);
-
- *out = (*in_buf) >> 19;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 10)) << (23 - 10);
- out = out.offset(1);
-
- *out = (*in_buf) >> 10;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 1)) << (23 - 1);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 1) % (1u32 << 23);
- out = out.offset(1);
- *out = (*in_buf) >> 24;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 15)) << (23 - 15);
- out = out.offset(1);
-
- *out = (*in_buf) >> 15;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 6)) << (23 - 6);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 6) % (1u32 << 23);
- out = out.offset(1);
- *out = (*in_buf) >> 29;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 20)) << (23 - 20);
- out = out.offset(1);
-
- *out = (*in_buf) >> 20;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 11)) << (23 - 11);
- out = out.offset(1);
-
- *out = (*in_buf) >> 11;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 2)) << (23 - 2);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 2) % (1u32 << 23);
- out = out.offset(1);
- *out = (*in_buf) >> 25;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 16)) << (23 - 16);
- out = out.offset(1);
-
- *out = (*in_buf) >> 16;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 7)) << (23 - 7);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 7) % (1u32 << 23);
- out = out.offset(1);
- *out = (*in_buf) >> 30;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 21)) << (23 - 21);
- out = out.offset(1);
-
- *out = (*in_buf) >> 21;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 12)) << (23 - 12);
- out = out.offset(1);
-
- *out = (*in_buf) >> 12;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 3)) << (23 - 3);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 3) % (1u32 << 23);
- out = out.offset(1);
- *out = (*in_buf) >> 26;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 17)) << (23 - 17);
- out = out.offset(1);
-
- *out = (*in_buf) >> 17;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 8)) << (23 - 8);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 8) % (1u32 << 23);
- out = out.offset(1);
- *out = (*in_buf) >> 31;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 22)) << (23 - 22);
- out = out.offset(1);
-
- *out = (*in_buf) >> 22;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 13)) << (23 - 13);
- out = out.offset(1);
-
- *out = (*in_buf) >> 13;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 4)) << (23 - 4);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 4) % (1u32 << 23);
- out = out.offset(1);
- *out = (*in_buf) >> 27;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 18)) << (23 - 18);
- out = out.offset(1);
-
- *out = (*in_buf) >> 18;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 9)) << (23 - 9);
- out = out.offset(1);
-
- *out = (*in_buf) >> 9;
-
- in_buf.offset(1)
-}
-
-unsafe fn unpack24_32(mut in_buf: *const u32, mut out: *mut u32) -> *const u32 {
- *out = (*in_buf) % (1u32 << 24);
- out = out.offset(1);
- *out = (*in_buf) >> 24;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 16)) << (24 - 16);
- out = out.offset(1);
-
- *out = (*in_buf) >> 16;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 8)) << (24 - 8);
- out = out.offset(1);
-
- *out = (*in_buf) >> 8;
- in_buf = in_buf.offset(1);
- out = out.offset(1);
-
- *out = (*in_buf) % (1u32 << 24);
- out = out.offset(1);
- *out = (*in_buf) >> 24;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 16)) << (24 - 16);
- out = out.offset(1);
-
- *out = (*in_buf) >> 16;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 8)) << (24 - 8);
- out = out.offset(1);
-
- *out = (*in_buf) >> 8;
- in_buf = in_buf.offset(1);
- out = out.offset(1);
-
- *out = (*in_buf) % (1u32 << 24);
- out = out.offset(1);
- *out = (*in_buf) >> 24;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 16)) << (24 - 16);
- out = out.offset(1);
-
- *out = (*in_buf) >> 16;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 8)) << (24 - 8);
- out = out.offset(1);
-
- *out = (*in_buf) >> 8;
- in_buf = in_buf.offset(1);
- out = out.offset(1);
-
- *out = (*in_buf) % (1u32 << 24);
- out = out.offset(1);
- *out = (*in_buf) >> 24;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 16)) << (24 - 16);
- out = out.offset(1);
-
- *out = (*in_buf) >> 16;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 8)) << (24 - 8);
- out = out.offset(1);
-
- *out = (*in_buf) >> 8;
- in_buf = in_buf.offset(1);
- out = out.offset(1);
-
- *out = (*in_buf) % (1u32 << 24);
- out = out.offset(1);
- *out = (*in_buf) >> 24;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 16)) << (24 - 16);
- out = out.offset(1);
-
- *out = (*in_buf) >> 16;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 8)) << (24 - 8);
- out = out.offset(1);
-
- *out = (*in_buf) >> 8;
- in_buf = in_buf.offset(1);
- out = out.offset(1);
-
- *out = (*in_buf) % (1u32 << 24);
- out = out.offset(1);
- *out = (*in_buf) >> 24;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 16)) << (24 - 16);
- out = out.offset(1);
-
- *out = (*in_buf) >> 16;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 8)) << (24 - 8);
- out = out.offset(1);
-
- *out = (*in_buf) >> 8;
- in_buf = in_buf.offset(1);
- out = out.offset(1);
-
- *out = (*in_buf) % (1u32 << 24);
- out = out.offset(1);
- *out = (*in_buf) >> 24;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 16)) << (24 - 16);
- out = out.offset(1);
-
- *out = (*in_buf) >> 16;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 8)) << (24 - 8);
- out = out.offset(1);
-
- *out = (*in_buf) >> 8;
- in_buf = in_buf.offset(1);
- out = out.offset(1);
-
- *out = (*in_buf) % (1u32 << 24);
- out = out.offset(1);
- *out = (*in_buf) >> 24;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 16)) << (24 - 16);
- out = out.offset(1);
-
- *out = (*in_buf) >> 16;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 8)) << (24 - 8);
- out = out.offset(1);
-
- *out = (*in_buf) >> 8;
-
- in_buf.offset(1)
-}
-
-unsafe fn unpack25_32(mut in_buf: *const u32, mut out: *mut u32) -> *const u32 {
- *out = (*in_buf) % (1u32 << 25);
- out = out.offset(1);
- *out = (*in_buf) >> 25;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 18)) << (25 - 18);
- out = out.offset(1);
-
- *out = (*in_buf) >> 18;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 11)) << (25 - 11);
- out = out.offset(1);
-
- *out = (*in_buf) >> 11;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 4)) << (25 - 4);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 4) % (1u32 << 25);
- out = out.offset(1);
- *out = (*in_buf) >> 29;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 22)) << (25 - 22);
- out = out.offset(1);
-
- *out = (*in_buf) >> 22;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 15)) << (25 - 15);
- out = out.offset(1);
-
- *out = (*in_buf) >> 15;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 8)) << (25 - 8);
- out = out.offset(1);
-
- *out = (*in_buf) >> 8;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 1)) << (25 - 1);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 1) % (1u32 << 25);
- out = out.offset(1);
- *out = (*in_buf) >> 26;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 19)) << (25 - 19);
- out = out.offset(1);
-
- *out = (*in_buf) >> 19;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 12)) << (25 - 12);
- out = out.offset(1);
-
- *out = (*in_buf) >> 12;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 5)) << (25 - 5);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 5) % (1u32 << 25);
- out = out.offset(1);
- *out = (*in_buf) >> 30;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 23)) << (25 - 23);
- out = out.offset(1);
-
- *out = (*in_buf) >> 23;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 16)) << (25 - 16);
- out = out.offset(1);
-
- *out = (*in_buf) >> 16;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 9)) << (25 - 9);
- out = out.offset(1);
-
- *out = (*in_buf) >> 9;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 2)) << (25 - 2);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 2) % (1u32 << 25);
- out = out.offset(1);
- *out = (*in_buf) >> 27;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 20)) << (25 - 20);
- out = out.offset(1);
-
- *out = (*in_buf) >> 20;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 13)) << (25 - 13);
- out = out.offset(1);
-
- *out = (*in_buf) >> 13;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 6)) << (25 - 6);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 6) % (1u32 << 25);
- out = out.offset(1);
- *out = (*in_buf) >> 31;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 24)) << (25 - 24);
- out = out.offset(1);
-
- *out = (*in_buf) >> 24;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 17)) << (25 - 17);
- out = out.offset(1);
-
- *out = (*in_buf) >> 17;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 10)) << (25 - 10);
- out = out.offset(1);
-
- *out = (*in_buf) >> 10;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 3)) << (25 - 3);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 3) % (1u32 << 25);
- out = out.offset(1);
- *out = (*in_buf) >> 28;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 21)) << (25 - 21);
- out = out.offset(1);
-
- *out = (*in_buf) >> 21;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 14)) << (25 - 14);
- out = out.offset(1);
-
- *out = (*in_buf) >> 14;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 7)) << (25 - 7);
- out = out.offset(1);
-
- *out = (*in_buf) >> 7;
-
- in_buf.offset(1)
-}
-
-unsafe fn unpack26_32(mut in_buf: *const u32, mut out: *mut u32) -> *const u32 {
- *out = (*in_buf) % (1u32 << 26);
- out = out.offset(1);
- *out = (*in_buf) >> 26;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 20)) << (26 - 20);
- out = out.offset(1);
-
- *out = (*in_buf) >> 20;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 14)) << (26 - 14);
- out = out.offset(1);
-
- *out = (*in_buf) >> 14;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 8)) << (26 - 8);
- out = out.offset(1);
-
- *out = (*in_buf) >> 8;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 2)) << (26 - 2);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 2) % (1u32 << 26);
- out = out.offset(1);
- *out = (*in_buf) >> 28;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 22)) << (26 - 22);
- out = out.offset(1);
-
- *out = (*in_buf) >> 22;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 16)) << (26 - 16);
- out = out.offset(1);
-
- *out = (*in_buf) >> 16;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 10)) << (26 - 10);
- out = out.offset(1);
-
- *out = (*in_buf) >> 10;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 4)) << (26 - 4);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 4) % (1u32 << 26);
- out = out.offset(1);
- *out = (*in_buf) >> 30;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 24)) << (26 - 24);
- out = out.offset(1);
-
- *out = (*in_buf) >> 24;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 18)) << (26 - 18);
- out = out.offset(1);
-
- *out = (*in_buf) >> 18;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 12)) << (26 - 12);
- out = out.offset(1);
-
- *out = (*in_buf) >> 12;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 6)) << (26 - 6);
- out = out.offset(1);
-
- *out = (*in_buf) >> 6;
- in_buf = in_buf.offset(1);
- out = out.offset(1);
-
- *out = (*in_buf) % (1u32 << 26);
- out = out.offset(1);
- *out = (*in_buf) >> 26;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 20)) << (26 - 20);
- out = out.offset(1);
-
- *out = (*in_buf) >> 20;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 14)) << (26 - 14);
- out = out.offset(1);
-
- *out = (*in_buf) >> 14;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 8)) << (26 - 8);
- out = out.offset(1);
-
- *out = (*in_buf) >> 8;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 2)) << (26 - 2);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 2) % (1u32 << 26);
- out = out.offset(1);
- *out = (*in_buf) >> 28;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 22)) << (26 - 22);
- out = out.offset(1);
-
- *out = (*in_buf) >> 22;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 16)) << (26 - 16);
- out = out.offset(1);
-
- *out = (*in_buf) >> 16;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 10)) << (26 - 10);
- out = out.offset(1);
-
- *out = (*in_buf) >> 10;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 4)) << (26 - 4);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 4) % (1u32 << 26);
- out = out.offset(1);
- *out = (*in_buf) >> 30;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 24)) << (26 - 24);
- out = out.offset(1);
-
- *out = (*in_buf) >> 24;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 18)) << (26 - 18);
- out = out.offset(1);
-
- *out = (*in_buf) >> 18;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 12)) << (26 - 12);
- out = out.offset(1);
-
- *out = (*in_buf) >> 12;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 6)) << (26 - 6);
- out = out.offset(1);
-
- *out = (*in_buf) >> 6;
-
- in_buf.offset(1)
-}
-
-unsafe fn unpack27_32(mut in_buf: *const u32, mut out: *mut u32) -> *const u32 {
- *out = (*in_buf) % (1u32 << 27);
- out = out.offset(1);
- *out = (*in_buf) >> 27;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 22)) << (27 - 22);
- out = out.offset(1);
-
- *out = (*in_buf) >> 22;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 17)) << (27 - 17);
- out = out.offset(1);
-
- *out = (*in_buf) >> 17;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 12)) << (27 - 12);
- out = out.offset(1);
-
- *out = (*in_buf) >> 12;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 7)) << (27 - 7);
- out = out.offset(1);
-
- *out = (*in_buf) >> 7;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 2)) << (27 - 2);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 2) % (1u32 << 27);
- out = out.offset(1);
- *out = (*in_buf) >> 29;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 24)) << (27 - 24);
- out = out.offset(1);
-
- *out = (*in_buf) >> 24;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 19)) << (27 - 19);
- out = out.offset(1);
-
- *out = (*in_buf) >> 19;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 14)) << (27 - 14);
- out = out.offset(1);
-
- *out = (*in_buf) >> 14;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 9)) << (27 - 9);
- out = out.offset(1);
-
- *out = (*in_buf) >> 9;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 4)) << (27 - 4);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 4) % (1u32 << 27);
- out = out.offset(1);
- *out = (*in_buf) >> 31;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 26)) << (27 - 26);
- out = out.offset(1);
-
- *out = (*in_buf) >> 26;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 21)) << (27 - 21);
- out = out.offset(1);
-
- *out = (*in_buf) >> 21;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 16)) << (27 - 16);
- out = out.offset(1);
-
- *out = (*in_buf) >> 16;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 11)) << (27 - 11);
- out = out.offset(1);
-
- *out = (*in_buf) >> 11;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 6)) << (27 - 6);
- out = out.offset(1);
-
- *out = (*in_buf) >> 6;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 1)) << (27 - 1);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 1) % (1u32 << 27);
- out = out.offset(1);
- *out = (*in_buf) >> 28;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 23)) << (27 - 23);
- out = out.offset(1);
-
- *out = (*in_buf) >> 23;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 18)) << (27 - 18);
- out = out.offset(1);
-
- *out = (*in_buf) >> 18;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 13)) << (27 - 13);
- out = out.offset(1);
-
- *out = (*in_buf) >> 13;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 8)) << (27 - 8);
- out = out.offset(1);
-
- *out = (*in_buf) >> 8;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 3)) << (27 - 3);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 3) % (1u32 << 27);
- out = out.offset(1);
- *out = (*in_buf) >> 30;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 25)) << (27 - 25);
- out = out.offset(1);
-
- *out = (*in_buf) >> 25;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 20)) << (27 - 20);
- out = out.offset(1);
-
- *out = (*in_buf) >> 20;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 15)) << (27 - 15);
- out = out.offset(1);
-
- *out = (*in_buf) >> 15;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 10)) << (27 - 10);
- out = out.offset(1);
-
- *out = (*in_buf) >> 10;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 5)) << (27 - 5);
- out = out.offset(1);
-
- *out = (*in_buf) >> 5;
-
- in_buf.offset(1)
-}
-
-unsafe fn unpack28_32(mut in_buf: *const u32, mut out: *mut u32) -> *const u32 {
- *out = (*in_buf) % (1u32 << 28);
- out = out.offset(1);
- *out = (*in_buf) >> 28;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 24)) << (28 - 24);
- out = out.offset(1);
-
- *out = (*in_buf) >> 24;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 20)) << (28 - 20);
- out = out.offset(1);
-
- *out = (*in_buf) >> 20;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 16)) << (28 - 16);
- out = out.offset(1);
-
- *out = (*in_buf) >> 16;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 12)) << (28 - 12);
- out = out.offset(1);
-
- *out = (*in_buf) >> 12;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 8)) << (28 - 8);
- out = out.offset(1);
-
- *out = (*in_buf) >> 8;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 4)) << (28 - 4);
- out = out.offset(1);
-
- *out = (*in_buf) >> 4;
- in_buf = in_buf.offset(1);
- out = out.offset(1);
-
- *out = (*in_buf) % (1u32 << 28);
- out = out.offset(1);
- *out = (*in_buf) >> 28;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 24)) << (28 - 24);
- out = out.offset(1);
-
- *out = (*in_buf) >> 24;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 20)) << (28 - 20);
- out = out.offset(1);
-
- *out = (*in_buf) >> 20;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 16)) << (28 - 16);
- out = out.offset(1);
-
- *out = (*in_buf) >> 16;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 12)) << (28 - 12);
- out = out.offset(1);
-
- *out = (*in_buf) >> 12;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 8)) << (28 - 8);
- out = out.offset(1);
-
- *out = (*in_buf) >> 8;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 4)) << (28 - 4);
- out = out.offset(1);
-
- *out = (*in_buf) >> 4;
- in_buf = in_buf.offset(1);
- out = out.offset(1);
-
- *out = (*in_buf) % (1u32 << 28);
- out = out.offset(1);
- *out = (*in_buf) >> 28;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 24)) << (28 - 24);
- out = out.offset(1);
-
- *out = (*in_buf) >> 24;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 20)) << (28 - 20);
- out = out.offset(1);
-
- *out = (*in_buf) >> 20;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 16)) << (28 - 16);
- out = out.offset(1);
-
- *out = (*in_buf) >> 16;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 12)) << (28 - 12);
- out = out.offset(1);
-
- *out = (*in_buf) >> 12;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 8)) << (28 - 8);
- out = out.offset(1);
-
- *out = (*in_buf) >> 8;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 4)) << (28 - 4);
- out = out.offset(1);
-
- *out = (*in_buf) >> 4;
- in_buf = in_buf.offset(1);
- out = out.offset(1);
-
- *out = (*in_buf) % (1u32 << 28);
- out = out.offset(1);
- *out = (*in_buf) >> 28;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 24)) << (28 - 24);
- out = out.offset(1);
-
- *out = (*in_buf) >> 24;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 20)) << (28 - 20);
- out = out.offset(1);
-
- *out = (*in_buf) >> 20;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 16)) << (28 - 16);
- out = out.offset(1);
-
- *out = (*in_buf) >> 16;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 12)) << (28 - 12);
- out = out.offset(1);
-
- *out = (*in_buf) >> 12;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 8)) << (28 - 8);
- out = out.offset(1);
-
- *out = (*in_buf) >> 8;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 4)) << (28 - 4);
- out = out.offset(1);
-
- *out = (*in_buf) >> 4;
-
- in_buf.offset(1)
-}
-
-unsafe fn unpack29_32(mut in_buf: *const u32, mut out: *mut u32) -> *const u32 {
- *out = (*in_buf) % (1u32 << 29);
- out = out.offset(1);
- *out = (*in_buf) >> 29;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 26)) << (29 - 26);
- out = out.offset(1);
-
- *out = (*in_buf) >> 26;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 23)) << (29 - 23);
- out = out.offset(1);
-
- *out = (*in_buf) >> 23;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 20)) << (29 - 20);
- out = out.offset(1);
-
- *out = (*in_buf) >> 20;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 17)) << (29 - 17);
- out = out.offset(1);
-
- *out = (*in_buf) >> 17;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 14)) << (29 - 14);
- out = out.offset(1);
-
- *out = (*in_buf) >> 14;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 11)) << (29 - 11);
- out = out.offset(1);
-
- *out = (*in_buf) >> 11;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 8)) << (29 - 8);
- out = out.offset(1);
-
- *out = (*in_buf) >> 8;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 5)) << (29 - 5);
- out = out.offset(1);
-
- *out = (*in_buf) >> 5;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 2)) << (29 - 2);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 2) % (1u32 << 29);
- out = out.offset(1);
- *out = (*in_buf) >> 31;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 28)) << (29 - 28);
- out = out.offset(1);
-
- *out = (*in_buf) >> 28;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 25)) << (29 - 25);
- out = out.offset(1);
-
- *out = (*in_buf) >> 25;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 22)) << (29 - 22);
- out = out.offset(1);
-
- *out = (*in_buf) >> 22;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 19)) << (29 - 19);
- out = out.offset(1);
-
- *out = (*in_buf) >> 19;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 16)) << (29 - 16);
- out = out.offset(1);
-
- *out = (*in_buf) >> 16;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 13)) << (29 - 13);
- out = out.offset(1);
-
- *out = (*in_buf) >> 13;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 10)) << (29 - 10);
- out = out.offset(1);
-
- *out = (*in_buf) >> 10;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 7)) << (29 - 7);
- out = out.offset(1);
-
- *out = (*in_buf) >> 7;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 4)) << (29 - 4);
- out = out.offset(1);
-
- *out = (*in_buf) >> 4;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 1)) << (29 - 1);
- out = out.offset(1);
-
- *out = ((*in_buf) >> 1) % (1u32 << 29);
- out = out.offset(1);
- *out = (*in_buf) >> 30;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 27)) << (29 - 27);
- out = out.offset(1);
-
- *out = (*in_buf) >> 27;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 24)) << (29 - 24);
- out = out.offset(1);
-
- *out = (*in_buf) >> 24;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 21)) << (29 - 21);
- out = out.offset(1);
-
- *out = (*in_buf) >> 21;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 18)) << (29 - 18);
- out = out.offset(1);
-
- *out = (*in_buf) >> 18;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 15)) << (29 - 15);
- out = out.offset(1);
-
- *out = (*in_buf) >> 15;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 12)) << (29 - 12);
- out = out.offset(1);
-
- *out = (*in_buf) >> 12;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 9)) << (29 - 9);
- out = out.offset(1);
-
- *out = (*in_buf) >> 9;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 6)) << (29 - 6);
- out = out.offset(1);
-
- *out = (*in_buf) >> 6;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 3)) << (29 - 3);
- out = out.offset(1);
-
- *out = (*in_buf) >> 3;
-
- in_buf.offset(1)
-}
-
-unsafe fn unpack30_32(mut in_buf: *const u32, mut out: *mut u32) -> *const u32 {
- *out = (*in_buf) % (1u32 << 30);
- out = out.offset(1);
- *out = (*in_buf) >> 30;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 28)) << (30 - 28);
- out = out.offset(1);
-
- *out = (*in_buf) >> 28;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 26)) << (30 - 26);
- out = out.offset(1);
-
- *out = (*in_buf) >> 26;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 24)) << (30 - 24);
- out = out.offset(1);
-
- *out = (*in_buf) >> 24;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 22)) << (30 - 22);
- out = out.offset(1);
-
- *out = (*in_buf) >> 22;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 20)) << (30 - 20);
- out = out.offset(1);
-
- *out = (*in_buf) >> 20;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 18)) << (30 - 18);
- out = out.offset(1);
-
- *out = (*in_buf) >> 18;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 16)) << (30 - 16);
- out = out.offset(1);
-
- *out = (*in_buf) >> 16;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 14)) << (30 - 14);
- out = out.offset(1);
-
- *out = (*in_buf) >> 14;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 12)) << (30 - 12);
- out = out.offset(1);
-
- *out = (*in_buf) >> 12;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 10)) << (30 - 10);
- out = out.offset(1);
-
- *out = (*in_buf) >> 10;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 8)) << (30 - 8);
- out = out.offset(1);
-
- *out = (*in_buf) >> 8;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 6)) << (30 - 6);
- out = out.offset(1);
-
- *out = (*in_buf) >> 6;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 4)) << (30 - 4);
- out = out.offset(1);
-
- *out = (*in_buf) >> 4;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 2)) << (30 - 2);
- out = out.offset(1);
-
- *out = (*in_buf) >> 2;
- in_buf = in_buf.offset(1);
- out = out.offset(1);
-
- *out = (*in_buf) % (1u32 << 30);
- out = out.offset(1);
- *out = (*in_buf) >> 30;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 28)) << (30 - 28);
- out = out.offset(1);
-
- *out = (*in_buf) >> 28;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 26)) << (30 - 26);
- out = out.offset(1);
-
- *out = (*in_buf) >> 26;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 24)) << (30 - 24);
- out = out.offset(1);
-
- *out = (*in_buf) >> 24;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 22)) << (30 - 22);
- out = out.offset(1);
-
- *out = (*in_buf) >> 22;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 20)) << (30 - 20);
- out = out.offset(1);
-
- *out = (*in_buf) >> 20;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 18)) << (30 - 18);
- out = out.offset(1);
-
- *out = (*in_buf) >> 18;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 16)) << (30 - 16);
- out = out.offset(1);
-
- *out = (*in_buf) >> 16;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 14)) << (30 - 14);
- out = out.offset(1);
-
- *out = (*in_buf) >> 14;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 12)) << (30 - 12);
- out = out.offset(1);
-
- *out = (*in_buf) >> 12;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 10)) << (30 - 10);
- out = out.offset(1);
-
- *out = (*in_buf) >> 10;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 8)) << (30 - 8);
- out = out.offset(1);
-
- *out = (*in_buf) >> 8;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 6)) << (30 - 6);
- out = out.offset(1);
-
- *out = (*in_buf) >> 6;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 4)) << (30 - 4);
- out = out.offset(1);
-
- *out = (*in_buf) >> 4;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 2)) << (30 - 2);
- out = out.offset(1);
-
- *out = (*in_buf) >> 2;
-
- in_buf.offset(1)
-}
-
-unsafe fn unpack31_32(mut in_buf: *const u32, mut out: *mut u32) -> *const u32 {
- *out = (*in_buf) % (1u32 << 31);
- out = out.offset(1);
- *out = (*in_buf) >> 31;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 30)) << (31 - 30);
- out = out.offset(1);
-
- *out = (*in_buf) >> 30;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 29)) << (31 - 29);
- out = out.offset(1);
-
- *out = (*in_buf) >> 29;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 28)) << (31 - 28);
- out = out.offset(1);
-
- *out = (*in_buf) >> 28;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 27)) << (31 - 27);
- out = out.offset(1);
-
- *out = (*in_buf) >> 27;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 26)) << (31 - 26);
- out = out.offset(1);
-
- *out = (*in_buf) >> 26;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 25)) << (31 - 25);
- out = out.offset(1);
-
- *out = (*in_buf) >> 25;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 24)) << (31 - 24);
- out = out.offset(1);
-
- *out = (*in_buf) >> 24;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 23)) << (31 - 23);
- out = out.offset(1);
-
- *out = (*in_buf) >> 23;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 22)) << (31 - 22);
- out = out.offset(1);
-
- *out = (*in_buf) >> 22;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 21)) << (31 - 21);
- out = out.offset(1);
-
- *out = (*in_buf) >> 21;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 20)) << (31 - 20);
- out = out.offset(1);
-
- *out = (*in_buf) >> 20;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 19)) << (31 - 19);
- out = out.offset(1);
-
- *out = (*in_buf) >> 19;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 18)) << (31 - 18);
- out = out.offset(1);
-
- *out = (*in_buf) >> 18;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 17)) << (31 - 17);
- out = out.offset(1);
-
- *out = (*in_buf) >> 17;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 16)) << (31 - 16);
- out = out.offset(1);
-
- *out = (*in_buf) >> 16;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 15)) << (31 - 15);
- out = out.offset(1);
-
- *out = (*in_buf) >> 15;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 14)) << (31 - 14);
- out = out.offset(1);
-
- *out = (*in_buf) >> 14;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 13)) << (31 - 13);
- out = out.offset(1);
-
- *out = (*in_buf) >> 13;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 12)) << (31 - 12);
- out = out.offset(1);
-
- *out = (*in_buf) >> 12;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 11)) << (31 - 11);
- out = out.offset(1);
-
- *out = (*in_buf) >> 11;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 10)) << (31 - 10);
- out = out.offset(1);
-
- *out = (*in_buf) >> 10;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 9)) << (31 - 9);
- out = out.offset(1);
-
- *out = (*in_buf) >> 9;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 8)) << (31 - 8);
- out = out.offset(1);
-
- *out = (*in_buf) >> 8;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 7)) << (31 - 7);
- out = out.offset(1);
-
- *out = (*in_buf) >> 7;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 6)) << (31 - 6);
- out = out.offset(1);
-
- *out = (*in_buf) >> 6;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 5)) << (31 - 5);
- out = out.offset(1);
-
- *out = (*in_buf) >> 5;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 4)) << (31 - 4);
- out = out.offset(1);
-
- *out = (*in_buf) >> 4;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 3)) << (31 - 3);
- out = out.offset(1);
-
- *out = (*in_buf) >> 3;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 2)) << (31 - 2);
- out = out.offset(1);
-
- *out = (*in_buf) >> 2;
- in_buf = in_buf.offset(1);
- *out |= ((*in_buf) % (1u32 << 1)) << (31 - 1);
- out = out.offset(1);
-
- *out = (*in_buf) >> 1;
-
- in_buf.offset(1)
-}
-
-unsafe fn unpack32_32(mut in_buf: *const u32, mut out: *mut u32) -> *const u32 {
- *out = *in_buf;
- in_buf = in_buf.offset(1);
- out = out.offset(1);
-
- *out = *in_buf;
- in_buf = in_buf.offset(1);
- out = out.offset(1);
-
- *out = *in_buf;
- in_buf = in_buf.offset(1);
- out = out.offset(1);
-
- *out = *in_buf;
- in_buf = in_buf.offset(1);
- out = out.offset(1);
-
- *out = *in_buf;
- in_buf = in_buf.offset(1);
- out = out.offset(1);
-
- *out = *in_buf;
- in_buf = in_buf.offset(1);
- out = out.offset(1);
-
- *out = *in_buf;
- in_buf = in_buf.offset(1);
- out = out.offset(1);
-
- *out = *in_buf;
- in_buf = in_buf.offset(1);
- out = out.offset(1);
-
- *out = *in_buf;
- in_buf = in_buf.offset(1);
- out = out.offset(1);
-
- *out = *in_buf;
- in_buf = in_buf.offset(1);
- out = out.offset(1);
-
- *out = *in_buf;
- in_buf = in_buf.offset(1);
- out = out.offset(1);
-
- *out = *in_buf;
- in_buf = in_buf.offset(1);
- out = out.offset(1);
-
- *out = *in_buf;
- in_buf = in_buf.offset(1);
- out = out.offset(1);
-
- *out = *in_buf;
- in_buf = in_buf.offset(1);
- out = out.offset(1);
-
- *out = *in_buf;
- in_buf = in_buf.offset(1);
- out = out.offset(1);
-
- *out = *in_buf;
- in_buf = in_buf.offset(1);
- out = out.offset(1);
-
- *out = *in_buf;
- in_buf = in_buf.offset(1);
- out = out.offset(1);
-
- *out = *in_buf;
- in_buf = in_buf.offset(1);
- out = out.offset(1);
-
- *out = *in_buf;
- in_buf = in_buf.offset(1);
- out = out.offset(1);
-
- *out = *in_buf;
- in_buf = in_buf.offset(1);
- out = out.offset(1);
-
- *out = *in_buf;
- in_buf = in_buf.offset(1);
- out = out.offset(1);
-
- *out = *in_buf;
- in_buf = in_buf.offset(1);
- out = out.offset(1);
-
- *out = *in_buf;
- in_buf = in_buf.offset(1);
- out = out.offset(1);
-
- *out = *in_buf;
- in_buf = in_buf.offset(1);
- out = out.offset(1);
-
- *out = *in_buf;
- in_buf = in_buf.offset(1);
- out = out.offset(1);
-
- *out = *in_buf;
- in_buf = in_buf.offset(1);
- out = out.offset(1);
-
- *out = *in_buf;
- in_buf = in_buf.offset(1);
- out = out.offset(1);
-
- *out = *in_buf;
- in_buf = in_buf.offset(1);
- out = out.offset(1);
-
- *out = *in_buf;
- in_buf = in_buf.offset(1);
- out = out.offset(1);
-
- *out = *in_buf;
- in_buf = in_buf.offset(1);
- out = out.offset(1);
-
- *out = *in_buf;
- in_buf = in_buf.offset(1);
- out = out.offset(1);
-
- *out = *in_buf;
-
- in_buf.offset(1)
-}
diff --git a/parquet/src/util/bit_util.rs b/parquet/src/util/bit_util.rs
deleted file mode 100644
index 45cfe2b..0000000
--- a/parquet/src/util/bit_util.rs
+++ /dev/null
@@ -1,1150 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-use std::{cmp, mem::size_of};
-
-use crate::data_type::AsBytes;
-use crate::errors::{ParquetError, Result};
-use crate::util::{bit_packing::unpack32, memory::ByteBufferPtr};
-
-#[inline]
-pub fn from_ne_slice<T: FromBytes>(bs: &[u8]) -> T {
- let mut b = T::Buffer::default();
- {
- let b = b.as_mut();
- let bs = &bs[..b.len()];
- b.copy_from_slice(bs);
- }
- T::from_ne_bytes(b)
-}
-
-pub trait FromBytes: Sized {
- type Buffer: AsMut<[u8]> + Default;
- fn from_le_bytes(bs: Self::Buffer) -> Self;
- fn from_be_bytes(bs: Self::Buffer) -> Self;
- fn from_ne_bytes(bs: Self::Buffer) -> Self;
-}
-
-macro_rules! from_le_bytes {
- ($($ty: ty),*) => {
- $(
- impl FromBytes for $ty {
- type Buffer = [u8; size_of::<Self>()];
- fn from_le_bytes(bs: Self::Buffer) -> Self {
- <$ty>::from_le_bytes(bs)
- }
- fn from_be_bytes(bs: Self::Buffer) -> Self {
- <$ty>::from_be_bytes(bs)
- }
- fn from_ne_bytes(bs: Self::Buffer) -> Self {
- <$ty>::from_ne_bytes(bs)
- }
- }
- )*
- };
-}
-
-impl FromBytes for bool {
- type Buffer = [u8; 1];
- fn from_le_bytes(bs: Self::Buffer) -> Self {
- Self::from_ne_bytes(bs)
- }
- fn from_be_bytes(bs: Self::Buffer) -> Self {
- Self::from_ne_bytes(bs)
- }
- fn from_ne_bytes(bs: Self::Buffer) -> Self {
- match bs[0] {
- 0 => false,
- 1 => true,
- _ => panic!("Invalid byte when reading bool"),
- }
- }
-}
-
-from_le_bytes! { u8, u16, u32, u64, i8, i16, i32, i64, f32, f64 }
-
-/// Reads `$size` of bytes from `$src`, and reinterprets them as type `$ty`, in
-/// little-endian order. `$ty` must implement the `Default` trait. Otherwise this won't
-/// compile.
-/// This is copied and modified from byteorder crate.
-macro_rules! read_num_bytes {
- ($ty:ty, $size:expr, $src:expr) => {{
- assert!($size <= $src.len());
- let mut buffer = <$ty as $crate::util::bit_util::FromBytes>::Buffer::default();
- buffer.as_mut()[..$size].copy_from_slice(&$src[..$size]);
- <$ty>::from_ne_bytes(buffer)
- }};
-}
-
-/// Converts value `val` of type `T` to a byte vector, by reading `num_bytes` from `val`.
-/// NOTE: if `val` is less than the size of `T` then it can be truncated.
-#[inline]
-pub fn convert_to_bytes<T>(val: &T, num_bytes: usize) -> Vec<u8>
-where
- T: ?Sized + AsBytes,
-{
- let mut bytes: Vec<u8> = vec![0; num_bytes];
- memcpy_value(val.as_bytes(), num_bytes, &mut bytes);
- bytes
-}
-
-#[inline]
-pub fn memcpy(source: &[u8], target: &mut [u8]) {
- assert!(target.len() >= source.len());
- target[..source.len()].copy_from_slice(source)
-}
-
-#[inline]
-pub fn memcpy_value<T>(source: &T, num_bytes: usize, target: &mut [u8])
-where
- T: ?Sized + AsBytes,
-{
- assert!(
- target.len() >= num_bytes,
- "Not enough space. Only had {} bytes but need to put {} bytes",
- target.len(),
- num_bytes
- );
- memcpy(&source.as_bytes()[..num_bytes], target)
-}
-
-/// Returns the ceil of value/divisor
-#[inline]
-pub fn ceil(value: i64, divisor: i64) -> i64 {
- value / divisor + ((value % divisor != 0) as i64)
-}
-
-/// Returns ceil(log2(x))
-#[inline]
-pub fn log2(mut x: u64) -> i32 {
- if x == 1 {
- return 0;
- }
- x -= 1;
- let mut result = 0;
- while x > 0 {
- x >>= 1;
- result += 1;
- }
- result
-}
-
-/// Returns the `num_bits` least-significant bits of `v`
-#[inline]
-pub fn trailing_bits(v: u64, num_bits: usize) -> u64 {
- if num_bits == 0 {
- return 0;
- }
- if num_bits >= 64 {
- return v;
- }
- let n = 64 - num_bits;
- (v << n) >> n
-}
-
-#[inline]
-pub fn set_array_bit(bits: &mut [u8], i: usize) {
- bits[i / 8] |= 1 << (i % 8);
-}
-
-#[inline]
-pub fn unset_array_bit(bits: &mut [u8], i: usize) {
- bits[i / 8] &= !(1 << (i % 8));
-}
-
-/// Returns the minimum number of bits needed to represent the value 'x'
-#[inline]
-pub fn num_required_bits(x: u64) -> usize {
- for i in (0..64).rev() {
- if x & (1u64 << i) != 0 {
- return i + 1;
- }
- }
- 0
-}
-
-static BIT_MASK: [u8; 8] = [1, 2, 4, 8, 16, 32, 64, 128];
-
-/// Returns whether bit at position `i` in `data` is set or not
-#[inline]
-pub fn get_bit(data: &[u8], i: usize) -> bool {
- (data[i >> 3] & BIT_MASK[i & 7]) != 0
-}
-
-/// Utility class for writing bit/byte streams. This class can write data in either
-/// bit packed or byte aligned fashion.
-pub struct BitWriter {
- buffer: Vec<u8>,
- max_bytes: usize,
- buffered_values: u64,
- byte_offset: usize,
- bit_offset: usize,
- start: usize,
-}
-
-impl BitWriter {
- pub fn new(max_bytes: usize) -> Self {
- Self {
- buffer: vec![0; max_bytes],
- max_bytes,
- buffered_values: 0,
- byte_offset: 0,
- bit_offset: 0,
- start: 0,
- }
- }
-
- /// Initializes the writer from the existing buffer `buffer` and starting
- /// offset `start`.
- pub fn new_from_buf(buffer: Vec<u8>, start: usize) -> Self {
- assert!(start < buffer.len());
- let len = buffer.len();
- Self {
- buffer,
- max_bytes: len,
- buffered_values: 0,
- byte_offset: start,
- bit_offset: 0,
- start,
- }
- }
-
- /// Extend buffer size
- #[inline]
- pub fn extend(&mut self, increment: usize) {
- self.max_bytes += increment;
- let extra = vec![0; increment];
- self.buffer.extend(extra);
- }
-
- /// Report buffer size
- #[inline]
- pub fn capacity(&mut self) -> usize {
- self.max_bytes
- }
-
- /// Consumes and returns the current buffer.
- #[inline]
- pub fn consume(mut self) -> Vec<u8> {
- self.flush();
- self.buffer.truncate(self.byte_offset);
- self.buffer
- }
-
- /// Flushes the internal buffered bits and returns the buffer's content.
- /// This is a borrow equivalent of `consume` method.
- #[inline]
- pub fn flush_buffer(&mut self) -> &[u8] {
- self.flush();
- &self.buffer()[0..self.byte_offset]
- }
-
- /// Clears the internal state so the buffer can be reused.
- #[inline]
- pub fn clear(&mut self) {
- self.buffered_values = 0;
- self.byte_offset = self.start;
- self.bit_offset = 0;
- }
-
- /// Flushes the internal buffered bits and the align the buffer to the next byte.
- #[inline]
- pub fn flush(&mut self) {
- let num_bytes = ceil(self.bit_offset as i64, 8) as usize;
- assert!(self.byte_offset + num_bytes <= self.max_bytes);
- memcpy_value(
- &self.buffered_values,
- num_bytes,
- &mut self.buffer[self.byte_offset..],
- );
- self.buffered_values = 0;
- self.bit_offset = 0;
- self.byte_offset += num_bytes;
- }
-
- /// Advances the current offset by skipping `num_bytes`, flushing the internal bit
- /// buffer first.
- /// This is useful when you want to jump over `num_bytes` bytes and come back later
- /// to fill these bytes.
- ///
- /// Returns error if `num_bytes` is beyond the boundary of the internal buffer.
- /// Otherwise, returns the old offset.
- #[inline]
- pub fn skip(&mut self, num_bytes: usize) -> Result<usize> {
- self.flush();
- assert!(self.byte_offset <= self.max_bytes);
- if self.byte_offset + num_bytes > self.max_bytes {
- return Err(general_err!(
- "Not enough bytes left in BitWriter. Need {} but only have {}",
- self.byte_offset + num_bytes,
- self.max_bytes
- ));
- }
- let result = self.byte_offset;
- self.byte_offset += num_bytes;
- Ok(result)
- }
-
- /// Returns a slice containing the next `num_bytes` bytes starting from the current
- /// offset, and advances the underlying buffer by `num_bytes`.
- /// This is useful when you want to jump over `num_bytes` bytes and come back later
- /// to fill these bytes.
- #[inline]
- pub fn get_next_byte_ptr(&mut self, num_bytes: usize) -> Result<&mut [u8]> {
- let offset = self.skip(num_bytes)?;
- Ok(&mut self.buffer[offset..offset + num_bytes])
- }
-
- #[inline]
- pub fn bytes_written(&self) -> usize {
- self.byte_offset - self.start + ceil(self.bit_offset as i64, 8) as usize
- }
-
- #[inline]
- pub fn buffer(&self) -> &[u8] {
- &self.buffer[self.start..]
- }
-
- #[inline]
- pub fn byte_offset(&self) -> usize {
- self.byte_offset
- }
-
- /// Returns the internal buffer length. This is the maximum number of bytes that this
- /// writer can write. User needs to call `consume` to consume the current buffer
- /// before more data can be written.
- #[inline]
- pub fn buffer_len(&self) -> usize {
- self.max_bytes
- }
-
- pub fn write_at(&mut self, offset: usize, value: u8) {
- self.buffer[offset] = value;
- }
-
- /// Writes the `num_bits` LSB of value `v` to the internal buffer of this writer.
- /// The `num_bits` must not be greater than 64. This is bit packed.
- ///
- /// Returns false if there's not enough room left. True otherwise.
- #[inline]
- pub fn put_value(&mut self, v: u64, num_bits: usize) -> bool {
- assert!(num_bits <= 64);
- assert_eq!(v.checked_shr(num_bits as u32).unwrap_or(0), 0); // covers case v >> 64
-
- if self.byte_offset * 8 + self.bit_offset + num_bits > self.max_bytes as usize * 8
- {
- return false;
- }
-
- self.buffered_values |= v << self.bit_offset;
- self.bit_offset += num_bits;
- if self.bit_offset >= 64 {
- memcpy_value(
- &self.buffered_values,
- 8,
- &mut self.buffer[self.byte_offset..],
- );
- self.byte_offset += 8;
- self.bit_offset -= 64;
- self.buffered_values = 0;
- // Perform checked right shift: v >> offset, where offset < 64, otherwise we
- // shift all bits
- self.buffered_values = v
- .checked_shr((num_bits - self.bit_offset) as u32)
- .unwrap_or(0);
- }
- assert!(self.bit_offset < 64);
- true
- }
-
- /// Writes `val` of `num_bytes` bytes to the next aligned byte. If size of `T` is
- /// larger than `num_bytes`, extra higher ordered bytes will be ignored.
- ///
- /// Returns false if there's not enough room left. True otherwise.
- #[inline]
- pub fn put_aligned<T: AsBytes>(&mut self, val: T, num_bytes: usize) -> bool {
- let result = self.get_next_byte_ptr(num_bytes);
- if result.is_err() {
- // TODO: should we return `Result` for this func?
- return false;
- }
- let mut ptr = result.unwrap();
- memcpy_value(&val, num_bytes, &mut ptr);
- true
- }
-
- /// Writes `val` of `num_bytes` bytes at the designated `offset`. The `offset` is the
- /// offset starting from the beginning of the internal buffer that this writer
- /// maintains. Note that this will overwrite any existing data between `offset` and
- /// `offset + num_bytes`. Also that if size of `T` is larger than `num_bytes`, extra
- /// higher ordered bytes will be ignored.
- ///
- /// Returns false if there's not enough room left, or the `pos` is not valid.
- /// True otherwise.
- #[inline]
- pub fn put_aligned_offset<T: AsBytes>(
- &mut self,
- val: T,
- num_bytes: usize,
- offset: usize,
- ) -> bool {
- if num_bytes + offset > self.max_bytes {
- return false;
- }
- memcpy_value(
- &val,
- num_bytes,
- &mut self.buffer[offset..offset + num_bytes],
- );
- true
- }
-
- /// Writes a VLQ encoded integer `v` to this buffer. The value is byte aligned.
- ///
- /// Returns false if there's not enough room left. True otherwise.
- #[inline]
- pub fn put_vlq_int(&mut self, mut v: u64) -> bool {
- let mut result = true;
- while v & 0xFFFFFFFFFFFFFF80 != 0 {
- result &= self.put_aligned::<u8>(((v & 0x7F) | 0x80) as u8, 1);
- v >>= 7;
- }
- result &= self.put_aligned::<u8>((v & 0x7F) as u8, 1);
- result
- }
-
- /// Writes a zigzag-VLQ encoded (in little endian order) int `v` to this buffer.
- /// Zigzag-VLQ is a variant of VLQ encoding where negative and positive
- /// numbers are encoded in a zigzag fashion.
- /// See: https://developers.google.com/protocol-buffers/docs/encoding
- ///
- /// Returns false if there's not enough room left. True otherwise.
- #[inline]
- pub fn put_zigzag_vlq_int(&mut self, v: i64) -> bool {
- let u: u64 = ((v << 1) ^ (v >> 63)) as u64;
- self.put_vlq_int(u)
- }
-}
-
-/// Maximum byte length for a VLQ encoded integer
-/// MAX_VLQ_BYTE_LEN = 5 for i32, and MAX_VLQ_BYTE_LEN = 10 for i64
-pub const MAX_VLQ_BYTE_LEN: usize = 10;
-
-pub struct BitReader {
- // The byte buffer to read from, passed in by client
- buffer: ByteBufferPtr,
-
- // Bytes are memcpy'd from `buffer` and values are read from this variable.
- // This is faster than reading values byte by byte directly from `buffer`
- buffered_values: u64,
-
- //
- // End Start
- // |............|B|B|B|B|B|B|B|B|..............|
- // ^ ^
- // bit_offset byte_offset
- //
- // Current byte offset in `buffer`
- byte_offset: usize,
-
- // Current bit offset in `buffered_values`
- bit_offset: usize,
-
- // Total number of bytes in `buffer`
- total_bytes: usize,
-}
-
-/// Utility class to read bit/byte stream. This class can read bits or bytes that are
-/// either byte aligned or not.
-impl BitReader {
- pub fn new(buffer: ByteBufferPtr) -> Self {
- let total_bytes = buffer.len();
- let num_bytes = cmp::min(8, total_bytes);
- let buffered_values = read_num_bytes!(u64, num_bytes, buffer.as_ref());
- BitReader {
- buffer,
- buffered_values,
- byte_offset: 0,
- bit_offset: 0,
- total_bytes,
- }
- }
-
- pub fn reset(&mut self, buffer: ByteBufferPtr) {
- self.buffer = buffer;
- self.total_bytes = self.buffer.len();
- let num_bytes = cmp::min(8, self.total_bytes);
- self.buffered_values = read_num_bytes!(u64, num_bytes, self.buffer.as_ref());
- self.byte_offset = 0;
- self.bit_offset = 0;
- }
-
- /// Gets the current byte offset
- #[inline]
- pub fn get_byte_offset(&self) -> usize {
- self.byte_offset + ceil(self.bit_offset as i64, 8) as usize
- }
-
- /// Reads a value of type `T` and of size `num_bits`.
- ///
- /// Returns `None` if there's not enough data available. `Some` otherwise.
- pub fn get_value<T: FromBytes>(&mut self, num_bits: usize) -> Option<T> {
- assert!(num_bits <= 64);
- assert!(num_bits <= size_of::<T>() * 8);
-
- if self.byte_offset * 8 + self.bit_offset + num_bits > self.total_bytes * 8 {
- return None;
- }
-
- let mut v = trailing_bits(self.buffered_values, self.bit_offset + num_bits)
- >> self.bit_offset;
- self.bit_offset += num_bits;
-
- if self.bit_offset >= 64 {
- self.byte_offset += 8;
- self.bit_offset -= 64;
-
- self.reload_buffer_values();
- v |= trailing_bits(self.buffered_values, self.bit_offset)
- .wrapping_shl((num_bits - self.bit_offset) as u32);
- }
-
- // TODO: better to avoid copying here
- Some(from_ne_slice(v.as_bytes()))
- }
-
- pub fn get_batch<T: FromBytes>(&mut self, batch: &mut [T], num_bits: usize) -> usize {
- assert!(num_bits <= 32);
- assert!(num_bits <= size_of::<T>() * 8);
-
- let mut values_to_read = batch.len();
- let needed_bits = num_bits * values_to_read;
- let remaining_bits = (self.total_bytes - self.byte_offset) * 8 - self.bit_offset;
- if remaining_bits < needed_bits {
- values_to_read = remaining_bits / num_bits;
- }
-
- let mut i = 0;
-
- // First align bit offset to byte offset
- if self.bit_offset != 0 {
- while i < values_to_read && self.bit_offset != 0 {
- batch[i] = self
- .get_value(num_bits)
- .expect("expected to have more data");
- i += 1;
- }
- }
-
- unsafe {
- let in_buf = &self.buffer.data()[self.byte_offset..];
- let mut in_ptr = in_buf as *const [u8] as *const u8 as *const u32;
- // FIXME assert!(memory::is_ptr_aligned(in_ptr));
- if size_of::<T>() == 4 {
- while values_to_read - i >= 32 {
- let out_ptr = &mut batch[i..] as *mut [T] as *mut T as *mut u32;
- in_ptr = unpack32(in_ptr, out_ptr, num_bits);
- self.byte_offset += 4 * num_bits;
- i += 32;
- }
- } else {
- let mut out_buf = [0u32; 32];
- let out_ptr = &mut out_buf as &mut [u32] as *mut [u32] as *mut u32;
- while values_to_read - i >= 32 {
- in_ptr = unpack32(in_ptr, out_ptr, num_bits);
- self.byte_offset += 4 * num_bits;
- for n in 0..32 {
- // We need to copy from smaller size to bigger size to avoid
- // overwriting other memory regions.
- if size_of::<T>() > size_of::<u32>() {
- std::ptr::copy_nonoverlapping(
- out_buf[n..].as_ptr() as *const u32,
- &mut batch[i] as *mut T as *mut u32,
- 1,
- );
- } else {
- std::ptr::copy_nonoverlapping(
- out_buf[n..].as_ptr() as *const T,
- &mut batch[i] as *mut T,
- 1,
- );
- }
- i += 1;
- }
- }
- }
- }
-
- assert!(values_to_read - i < 32);
-
- self.reload_buffer_values();
- while i < values_to_read {
- batch[i] = self
- .get_value(num_bits)
- .expect("expected to have more data");
- i += 1;
- }
-
- values_to_read
- }
-
- /// Reads a `num_bytes`-sized value from this buffer and return it.
- /// `T` needs to be a little-endian native type. The value is assumed to be byte
- /// aligned so the bit reader will be advanced to the start of the next byte before
- /// reading the value.
-
- /// Returns `Some` if there's enough bytes left to form a value of `T`.
- /// Otherwise `None`.
- pub fn get_aligned<T: FromBytes>(&mut self, num_bytes: usize) -> Option<T> {
- let bytes_read = ceil(self.bit_offset as i64, 8) as usize;
- if self.byte_offset + bytes_read + num_bytes > self.total_bytes {
- return None;
- }
-
- // Advance byte_offset to next unread byte and read num_bytes
- self.byte_offset += bytes_read;
- let v = read_num_bytes!(T, num_bytes, self.buffer.data()[self.byte_offset..]);
- self.byte_offset += num_bytes;
-
- // Reset buffered_values
- self.bit_offset = 0;
- self.reload_buffer_values();
- Some(v)
- }
-
- /// Reads a VLQ encoded (in little endian order) int from the stream.
- /// The encoded int must start at the beginning of a byte.
- ///
- /// Returns `None` if there's not enough bytes in the stream. `Some` otherwise.
- pub fn get_vlq_int(&mut self) -> Option<i64> {
- let mut shift = 0;
- let mut v: i64 = 0;
- while let Some(byte) = self.get_aligned::<u8>(1) {
- v |= ((byte & 0x7F) as i64) << shift;
- shift += 7;
- assert!(
- shift <= MAX_VLQ_BYTE_LEN * 7,
- "Num of bytes exceed MAX_VLQ_BYTE_LEN ({})",
- MAX_VLQ_BYTE_LEN
- );
- if byte & 0x80 == 0 {
- return Some(v);
- }
- }
- None
- }
-
- /// Reads a zigzag-VLQ encoded (in little endian order) int from the stream
- /// Zigzag-VLQ is a variant of VLQ encoding where negative and positive numbers are
- /// encoded in a zigzag fashion.
- /// See: https://developers.google.com/protocol-buffers/docs/encoding
- ///
- /// Note: the encoded int must start at the beginning of a byte.
- ///
- /// Returns `None` if the number of bytes there's not enough bytes in the stream.
- /// `Some` otherwise.
- #[inline]
- pub fn get_zigzag_vlq_int(&mut self) -> Option<i64> {
- self.get_vlq_int().map(|v| {
- let u = v as u64;
- (u >> 1) as i64 ^ -((u & 1) as i64)
- })
- }
-
- fn reload_buffer_values(&mut self) {
- let bytes_to_read = cmp::min(self.total_bytes - self.byte_offset, 8);
- self.buffered_values =
- read_num_bytes!(u64, bytes_to_read, self.buffer.data()[self.byte_offset..]);
- }
-}
-
-impl From<Vec<u8>> for BitReader {
- #[inline]
- fn from(buffer: Vec<u8>) -> Self {
- BitReader::new(ByteBufferPtr::new(buffer))
- }
-}
-
-#[cfg(test)]
-mod tests {
- use super::super::test_common::*;
- use super::*;
-
- use rand::distributions::{Distribution, Standard};
- use std::fmt::Debug;
-
- #[test]
- fn test_ceil() {
- assert_eq!(ceil(0, 1), 0);
- assert_eq!(ceil(1, 1), 1);
- assert_eq!(ceil(1, 2), 1);
- assert_eq!(ceil(1, 8), 1);
- assert_eq!(ceil(7, 8), 1);
- assert_eq!(ceil(8, 8), 1);
- assert_eq!(ceil(9, 8), 2);
- assert_eq!(ceil(9, 9), 1);
- assert_eq!(ceil(10000000000, 10), 1000000000);
- assert_eq!(ceil(10, 10000000000), 1);
- assert_eq!(ceil(10000000000, 1000000000), 10);
- }
-
- #[test]
- fn test_bit_reader_get_byte_offset() {
- let buffer = vec![255; 10];
- let mut bit_reader = BitReader::from(buffer);
- assert_eq!(bit_reader.get_byte_offset(), 0); // offset (0 bytes, 0 bits)
- bit_reader.get_value::<i32>(6);
- assert_eq!(bit_reader.get_byte_offset(), 1); // offset (0 bytes, 6 bits)
- bit_reader.get_value::<i32>(10);
- assert_eq!(bit_reader.get_byte_offset(), 2); // offset (0 bytes, 16 bits)
- bit_reader.get_value::<i32>(20);
- assert_eq!(bit_reader.get_byte_offset(), 5); // offset (0 bytes, 36 bits)
- bit_reader.get_value::<i32>(30);
- assert_eq!(bit_reader.get_byte_offset(), 9); // offset (8 bytes, 2 bits)
- }
-
- #[test]
- fn test_bit_reader_get_value() {
- let buffer = vec![255, 0];
- let mut bit_reader = BitReader::from(buffer);
- assert_eq!(bit_reader.get_value::<i32>(1), Some(1));
- assert_eq!(bit_reader.get_value::<i32>(2), Some(3));
- assert_eq!(bit_reader.get_value::<i32>(3), Some(7));
- assert_eq!(bit_reader.get_value::<i32>(4), Some(3));
- }
-
- #[test]
- fn test_bit_reader_get_value_boundary() {
- let buffer = vec![10, 0, 0, 0, 20, 0, 30, 0, 0, 0, 40, 0];
- let mut bit_reader = BitReader::from(buffer);
- assert_eq!(bit_reader.get_value::<i64>(32), Some(10));
- assert_eq!(bit_reader.get_value::<i64>(16), Some(20));
- assert_eq!(bit_reader.get_value::<i64>(32), Some(30));
- assert_eq!(bit_reader.get_value::<i64>(16), Some(40));
- }
-
- #[test]
- fn test_bit_reader_get_aligned() {
- // 01110101 11001011
- let buffer = ByteBufferPtr::new(vec![0x75, 0xCB]);
- let mut bit_reader = BitReader::new(buffer.all());
- assert_eq!(bit_reader.get_value::<i32>(3), Some(5));
- assert_eq!(bit_reader.get_aligned::<i32>(1), Some(203));
- assert_eq!(bit_reader.get_value::<i32>(1), None);
- bit_reader.reset(buffer.all());
- assert_eq!(bit_reader.get_aligned::<i32>(3), None);
- }
-
- #[test]
- fn test_bit_reader_get_vlq_int() {
- // 10001001 00000001 11110010 10110101 00000110
- let buffer: Vec<u8> = vec![0x89, 0x01, 0xF2, 0xB5, 0x06];
- let mut bit_reader = BitReader::from(buffer);
- assert_eq!(bit_reader.get_vlq_int(), Some(137));
- assert_eq!(bit_reader.get_vlq_int(), Some(105202));
- }
-
- #[test]
- fn test_bit_reader_get_zigzag_vlq_int() {
- let buffer: Vec<u8> = vec![0, 1, 2, 3];
- let mut bit_reader = BitReader::from(buffer);
- assert_eq!(bit_reader.get_zigzag_vlq_int(), Some(0));
- assert_eq!(bit_reader.get_zigzag_vlq_int(), Some(-1));
- assert_eq!(bit_reader.get_zigzag_vlq_int(), Some(1));
- assert_eq!(bit_reader.get_zigzag_vlq_int(), Some(-2));
- }
-
- #[test]
- fn test_set_array_bit() {
- let mut buffer = vec![0, 0, 0];
- set_array_bit(&mut buffer[..], 1);
- assert_eq!(buffer, vec![2, 0, 0]);
- set_array_bit(&mut buffer[..], 4);
- assert_eq!(buffer, vec![18, 0, 0]);
- unset_array_bit(&mut buffer[..], 1);
- assert_eq!(buffer, vec![16, 0, 0]);
- set_array_bit(&mut buffer[..], 10);
- assert_eq!(buffer, vec![16, 4, 0]);
- set_array_bit(&mut buffer[..], 10);
- assert_eq!(buffer, vec![16, 4, 0]);
- set_array_bit(&mut buffer[..], 11);
- assert_eq!(buffer, vec![16, 12, 0]);
- unset_array_bit(&mut buffer[..], 10);
- assert_eq!(buffer, vec![16, 8, 0]);
- }
-
- #[test]
- fn test_num_required_bits() {
- assert_eq!(num_required_bits(0), 0);
- assert_eq!(num_required_bits(1), 1);
- assert_eq!(num_required_bits(2), 2);
- assert_eq!(num_required_bits(4), 3);
- assert_eq!(num_required_bits(8), 4);
- assert_eq!(num_required_bits(10), 4);
- assert_eq!(num_required_bits(12), 4);
- assert_eq!(num_required_bits(16), 5);
- }
-
- #[test]
- fn test_get_bit() {
- // 00001101
- assert_eq!(true, get_bit(&[0b00001101], 0));
- assert_eq!(false, get_bit(&[0b00001101], 1));
- assert_eq!(true, get_bit(&[0b00001101], 2));
- assert_eq!(true, get_bit(&[0b00001101], 3));
-
- // 01001001 01010010
- assert_eq!(true, get_bit(&[0b01001001, 0b01010010], 0));
- assert_eq!(false, get_bit(&[0b01001001, 0b01010010], 1));
- assert_eq!(false, get_bit(&[0b01001001, 0b01010010], 2));
- assert_eq!(true, get_bit(&[0b01001001, 0b01010010], 3));
- assert_eq!(false, get_bit(&[0b01001001, 0b01010010], 4));
- assert_eq!(false, get_bit(&[0b01001001, 0b01010010], 5));
- assert_eq!(true, get_bit(&[0b01001001, 0b01010010], 6));
- assert_eq!(false, get_bit(&[0b01001001, 0b01010010], 7));
- assert_eq!(false, get_bit(&[0b01001001, 0b01010010], 8));
- assert_eq!(true, get_bit(&[0b01001001, 0b01010010], 9));
- assert_eq!(false, get_bit(&[0b01001001, 0b01010010], 10));
- assert_eq!(false, get_bit(&[0b01001001, 0b01010010], 11));
- assert_eq!(true, get_bit(&[0b01001001, 0b01010010], 12));
- assert_eq!(false, get_bit(&[0b01001001, 0b01010010], 13));
- assert_eq!(true, get_bit(&[0b01001001, 0b01010010], 14));
- assert_eq!(false, get_bit(&[0b01001001, 0b01010010], 15));
- }
-
- #[test]
- fn test_log2() {
- assert_eq!(log2(1), 0);
- assert_eq!(log2(2), 1);
- assert_eq!(log2(3), 2);
- assert_eq!(log2(4), 2);
- assert_eq!(log2(5), 3);
- assert_eq!(log2(5), 3);
- assert_eq!(log2(6), 3);
- assert_eq!(log2(7), 3);
- assert_eq!(log2(8), 3);
- assert_eq!(log2(9), 4);
- }
-
- #[test]
- fn test_skip() {
- let mut writer = BitWriter::new(5);
- let old_offset = writer.skip(1).expect("skip() should return OK");
- writer.put_aligned(42, 4);
- writer.put_aligned_offset(0x10, 1, old_offset);
- let result = writer.consume();
- assert_eq!(result.as_ref(), [0x10, 42, 0, 0, 0]);
-
- writer = BitWriter::new(4);
- let result = writer.skip(5);
- assert!(result.is_err());
- }
-
- #[test]
- fn test_get_next_byte_ptr() {
- let mut writer = BitWriter::new(5);
- {
- let first_byte = writer
- .get_next_byte_ptr(1)
- .expect("get_next_byte_ptr() should return OK");
- first_byte[0] = 0x10;
- }
- writer.put_aligned(42, 4);
- let result = writer.consume();
- assert_eq!(result.as_ref(), [0x10, 42, 0, 0, 0]);
- }
-
- #[test]
- fn test_consume_flush_buffer() {
- let mut writer1 = BitWriter::new(3);
- let mut writer2 = BitWriter::new(3);
- for i in 1..10 {
- writer1.put_value(i, 4);
- writer2.put_value(i, 4);
- }
- let res1 = writer1.flush_buffer();
- let res2 = writer2.consume();
- assert_eq!(res1, &res2[..]);
- }
-
- #[test]
- fn test_put_get_bool() {
- let len = 8;
- let mut writer = BitWriter::new(len);
-
- for i in 0..8 {
- let result = writer.put_value(i % 2, 1);
- assert!(result);
- }
-
- writer.flush();
- {
- let buffer = writer.buffer();
- assert_eq!(buffer[0], 0b10101010);
- }
-
- // Write 00110011
- for i in 0..8 {
- let result = match i {
- 0 | 1 | 4 | 5 => writer.put_value(false as u64, 1),
- _ => writer.put_value(true as u64, 1),
- };
- assert!(result);
- }
- writer.flush();
- {
- let buffer = writer.buffer();
- assert_eq!(buffer[0], 0b10101010);
- assert_eq!(buffer[1], 0b11001100);
- }
-
- let mut reader = BitReader::from(writer.consume());
-
- for i in 0..8 {
- let val = reader
- .get_value::<u8>(1)
- .expect("get_value() should return OK");
- assert_eq!(val, i % 2);
- }
-
- for i in 0..8 {
- let val = reader
- .get_value::<bool>(1)
- .expect("get_value() should return OK");
- match i {
- 0 | 1 | 4 | 5 => assert_eq!(val, false),
- _ => assert_eq!(val, true),
- }
- }
- }
-
- #[test]
- fn test_put_value_roundtrip() {
- test_put_value_rand_numbers(32, 2);
- test_put_value_rand_numbers(32, 3);
- test_put_value_rand_numbers(32, 4);
- test_put_value_rand_numbers(32, 5);
- test_put_value_rand_numbers(32, 6);
- test_put_value_rand_numbers(32, 7);
- test_put_value_rand_numbers(32, 8);
- test_put_value_rand_numbers(64, 16);
- test_put_value_rand_numbers(64, 24);
- test_put_value_rand_numbers(64, 32);
- }
-
- fn test_put_value_rand_numbers(total: usize, num_bits: usize) {
- assert!(num_bits < 64);
- let num_bytes = ceil(num_bits as i64, 8);
- let mut writer = BitWriter::new(num_bytes as usize * total);
- let values: Vec<u64> = random_numbers::<u64>(total)
- .iter()
- .map(|v| v & ((1 << num_bits) - 1))
- .collect();
- (0..total).for_each(|i| {
- assert!(
- writer.put_value(values[i] as u64, num_bits),
- "[{}]: put_value() failed",
- i
- );
- });
-
- let mut reader = BitReader::from(writer.consume());
- (0..total).for_each(|i| {
- let v = reader
- .get_value::<u64>(num_bits)
- .expect("get_value() should return OK");
- assert_eq!(
- v, values[i],
- "[{}]: expected {} but got {}",
- i, values[i], v
- );
- });
- }
-
- #[test]
- fn test_get_batch() {
- const SIZE: &[usize] = &[1, 31, 32, 33, 128, 129];
- for s in SIZE {
- for i in 0..33 {
- match i {
- 0..=8 => test_get_batch_helper::<u8>(*s, i),
- 9..=16 => test_get_batch_helper::<u16>(*s, i),
- _ => test_get_batch_helper::<u32>(*s, i),
- }
- }
- }
- }
-
- fn test_get_batch_helper<T>(total: usize, num_bits: usize)
- where
- T: FromBytes + Default + Clone + Debug + Eq,
- {
- assert!(num_bits <= 32);
- let num_bytes = ceil(num_bits as i64, 8);
- let mut writer = BitWriter::new(num_bytes as usize * total);
-
- let values: Vec<u32> = random_numbers::<u32>(total)
- .iter()
- .map(|v| v & ((1u64 << num_bits) - 1) as u32)
- .collect();
-
- // Generic values used to check against actual values read from `get_batch`.
- let expected_values: Vec<T> =
- values.iter().map(|v| from_ne_slice(v.as_bytes())).collect();
-
- (0..total).for_each(|i| {
- assert!(writer.put_value(values[i] as u64, num_bits));
- });
-
- let buf = writer.consume();
- let mut reader = BitReader::from(buf);
- let mut batch = vec![T::default(); values.len()];
- let values_read = reader.get_batch::<T>(&mut batch, num_bits);
- assert_eq!(values_read, values.len());
- for i in 0..batch.len() {
- assert_eq!(
- batch[i], expected_values[i],
- "num_bits = {}, index = {}",
- num_bits, i
- );
- }
- }
-
- #[test]
- fn test_put_aligned_roundtrip() {
- test_put_aligned_rand_numbers::<u8>(4, 3);
- test_put_aligned_rand_numbers::<u8>(16, 5);
- test_put_aligned_rand_numbers::<i16>(32, 7);
- test_put_aligned_rand_numbers::<i16>(32, 9);
- test_put_aligned_rand_numbers::<i32>(32, 11);
- test_put_aligned_rand_numbers::<i32>(32, 13);
- test_put_aligned_rand_numbers::<i64>(32, 17);
- test_put_aligned_rand_numbers::<i64>(32, 23);
- }
-
- fn test_put_aligned_rand_numbers<T>(total: usize, num_bits: usize)
- where
- T: Copy + FromBytes + AsBytes + Debug + PartialEq,
- Standard: Distribution<T>,
- {
- assert!(num_bits <= 32);
- assert!(total % 2 == 0);
-
- let aligned_value_byte_width = std::mem::size_of::<T>();
- let value_byte_width = ceil(num_bits as i64, 8) as usize;
- let mut writer =
- BitWriter::new((total / 2) * (aligned_value_byte_width + value_byte_width));
- let values: Vec<u32> = random_numbers::<u32>(total / 2)
- .iter()
- .map(|v| v & ((1 << num_bits) - 1))
- .collect();
- let aligned_values = random_numbers::<T>(total / 2);
-
- for i in 0..total {
- let j = i / 2;
- if i % 2 == 0 {
- assert!(
- writer.put_value(values[j] as u64, num_bits),
- "[{}]: put_value() failed",
- i
- );
- } else {
- assert!(
- writer.put_aligned::<T>(aligned_values[j], aligned_value_byte_width),
- "[{}]: put_aligned() failed",
- i
- );
- }
- }
-
- let mut reader = BitReader::from(writer.consume());
- for i in 0..total {
- let j = i / 2;
- if i % 2 == 0 {
- let v = reader
- .get_value::<u64>(num_bits)
- .expect("get_value() should return OK");
- assert_eq!(
- v, values[j] as u64,
- "[{}]: expected {} but got {}",
- i, values[j], v
- );
- } else {
- let v = reader
- .get_aligned::<T>(aligned_value_byte_width)
- .expect("get_aligned() should return OK");
- assert_eq!(
- v, aligned_values[j],
- "[{}]: expected {:?} but got {:?}",
- i, aligned_values[j], v
- );
- }
- }
- }
-
- #[test]
- fn test_put_vlq_int() {
- let total = 64;
- let mut writer = BitWriter::new(total * 32);
- let values = random_numbers::<u32>(total);
- (0..total).for_each(|i| {
- assert!(
- writer.put_vlq_int(values[i] as u64),
- "[{}]; put_vlq_int() failed",
- i
- );
- });
-
- let mut reader = BitReader::from(writer.consume());
- (0..total).for_each(|i| {
- let v = reader
- .get_vlq_int()
- .expect("get_vlq_int() should return OK");
- assert_eq!(
- v as u32, values[i],
- "[{}]: expected {} but got {}",
- i, values[i], v
- );
- });
- }
-
- #[test]
- fn test_put_zigzag_vlq_int() {
- let total = 64;
- let mut writer = BitWriter::new(total * 32);
- let values = random_numbers::<i32>(total);
- (0..total).for_each(|i| {
- assert!(
- writer.put_zigzag_vlq_int(values[i] as i64),
- "[{}]; put_zigzag_vlq_int() failed",
- i
- );
- });
-
- let mut reader = BitReader::from(writer.consume());
- (0..total).for_each(|i| {
- let v = reader
- .get_zigzag_vlq_int()
- .expect("get_zigzag_vlq_int() should return OK");
- assert_eq!(
- v as i32, values[i],
- "[{}]: expected {} but got {}",
- i, values[i], v
- );
- });
- }
-}
diff --git a/parquet/src/util/cursor.rs b/parquet/src/util/cursor.rs
deleted file mode 100644
index eaed6c7..0000000
--- a/parquet/src/util/cursor.rs
+++ /dev/null
@@ -1,261 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-use std::io::{self, Cursor, Error, ErrorKind, Read, Seek, SeekFrom, Write};
-use std::sync::{Arc, Mutex};
-use std::{cmp, fmt};
-
-use crate::file::writer::TryClone;
-
-/// This is object to use if your file is already in memory.
-/// The sliceable cursor is similar to std::io::Cursor, except that it makes it easy to create "cursor slices".
-/// To achieve this, it uses Arc instead of shared references. Indeed reference fields are painful
-/// because the lack of Generic Associated Type implies that you would require complex lifetime propagation when
-/// returning such a cursor.
-#[allow(clippy::rc_buffer)]
-pub struct SliceableCursor {
- inner: Arc<Vec<u8>>,
- start: u64,
- length: usize,
- pos: u64,
-}
-
-impl fmt::Debug for SliceableCursor {
- fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
- f.debug_struct("SliceableCursor")
- .field("start", &self.start)
- .field("length", &self.length)
- .field("pos", &self.pos)
- .field("inner.len", &self.inner.len())
- .finish()
- }
-}
-
-impl SliceableCursor {
- pub fn new(content: impl Into<Arc<Vec<u8>>>) -> Self {
- let inner = content.into();
- let size = inner.len();
- SliceableCursor {
- inner,
- start: 0,
- pos: 0,
- length: size,
- }
- }
-
- /// Create a slice cursor using the same data as a current one.
- pub fn slice(&self, start: u64, length: usize) -> io::Result<Self> {
- let new_start = self.start + start;
- if new_start >= self.inner.len() as u64
- || new_start as usize + length > self.inner.len()
- {
- return Err(Error::new(ErrorKind::InvalidInput, "out of bound"));
- }
- Ok(SliceableCursor {
- inner: Arc::clone(&self.inner),
- start: new_start,
- pos: new_start,
- length,
- })
- }
-
- fn remaining_slice(&self) -> &[u8] {
- let end = self.start as usize + self.length;
- let offset = cmp::min(self.pos, end as u64) as usize;
- &self.inner[offset..end]
- }
-
- /// Get the length of the current cursor slice
- pub fn len(&self) -> u64 {
- self.length as u64
- }
-
- /// return true if the cursor is empty (self.len() == 0)
- pub fn is_empty(&self) -> bool {
- self.len() == 0
- }
-}
-
-/// Implementation inspired by std::io::Cursor
-impl Read for SliceableCursor {
- fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
- let n = Read::read(&mut self.remaining_slice(), buf)?;
- self.pos += n as u64;
- Ok(n)
- }
-}
-
-impl Seek for SliceableCursor {
- fn seek(&mut self, pos: SeekFrom) -> io::Result<u64> {
- let new_pos = match pos {
- SeekFrom::Start(pos) => pos as i64,
- SeekFrom::End(pos) => self.inner.len() as i64 + pos as i64,
- SeekFrom::Current(pos) => self.pos as i64 + pos as i64,
- };
-
- if new_pos < 0 {
- Err(Error::new(
- ErrorKind::InvalidInput,
- format!(
- "Request out of bounds: cur position {} + seek {:?} < 0: {}",
- self.pos, pos, new_pos
- ),
- ))
- } else if new_pos >= self.inner.len() as i64 {
- Err(Error::new(
- ErrorKind::InvalidInput,
- format!(
- "Request out of bounds: cur position {} + seek {:?} >= length {}: {}",
- self.pos,
- pos,
- self.inner.len(),
- new_pos
- ),
- ))
- } else {
- self.pos = new_pos as u64;
- Ok(self.start)
- }
- }
-}
-
-/// Use this type to write Parquet to memory rather than a file.
-#[derive(Debug, Default, Clone)]
-pub struct InMemoryWriteableCursor {
- buffer: Arc<Mutex<Cursor<Vec<u8>>>>,
-}
-
-impl InMemoryWriteableCursor {
- /// Consume this instance and return the underlying buffer as long as there are no other
- /// references to this instance.
- pub fn into_inner(self) -> Option<Vec<u8>> {
- Arc::try_unwrap(self.buffer)
- .ok()
- .and_then(|mutex| mutex.into_inner().ok())
- .map(|cursor| cursor.into_inner())
- }
-
- /// Returns a clone of the underlying buffer
- pub fn data(&self) -> Vec<u8> {
- let inner = self.buffer.lock().unwrap();
- inner.get_ref().to_vec()
- }
-}
-
-impl TryClone for InMemoryWriteableCursor {
- fn try_clone(&self) -> std::io::Result<Self> {
- Ok(Self {
- buffer: self.buffer.clone(),
- })
- }
-}
-
-impl Write for InMemoryWriteableCursor {
- fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
- let mut inner = self.buffer.lock().unwrap();
- inner.write(buf)
- }
-
- fn flush(&mut self) -> std::io::Result<()> {
- let mut inner = self.buffer.lock().unwrap();
- inner.flush()
- }
-}
-
-impl Seek for InMemoryWriteableCursor {
- fn seek(&mut self, pos: SeekFrom) -> std::io::Result<u64> {
- let mut inner = self.buffer.lock().unwrap();
- inner.seek(pos)
- }
-}
-
-#[cfg(test)]
-mod tests {
- use super::*;
-
- /// Create a SliceableCursor of all u8 values in ascending order
- fn get_u8_range() -> SliceableCursor {
- let data: Vec<u8> = (0u8..=255).collect();
- SliceableCursor::new(data)
- }
-
- /// Reads all the bytes in the slice and checks that it matches the u8 range from start to end_included
- fn check_read_all(mut cursor: SliceableCursor, start: u8, end_included: u8) {
- let mut target = vec![];
- let cursor_res = cursor.read_to_end(&mut target);
- println!("{:?}", cursor_res);
- assert!(!cursor_res.is_err(), "reading error");
- assert_eq!((end_included - start) as usize + 1, cursor_res.unwrap());
- assert_eq!((start..=end_included).collect::<Vec<_>>(), target);
- }
-
- #[test]
- fn read_all_whole() {
- let cursor = get_u8_range();
- check_read_all(cursor, 0, 255);
- }
-
- #[test]
- fn read_all_slice() {
- let cursor = get_u8_range().slice(10, 10).expect("error while slicing");
- check_read_all(cursor, 10, 19);
- }
-
- #[test]
- fn seek_cursor_start() {
- let mut cursor = get_u8_range();
-
- cursor.seek(SeekFrom::Start(5)).unwrap();
- check_read_all(cursor, 5, 255);
- }
-
- #[test]
- fn seek_cursor_current() {
- let mut cursor = get_u8_range();
- cursor.seek(SeekFrom::Start(10)).unwrap();
- cursor.seek(SeekFrom::Current(10)).unwrap();
- check_read_all(cursor, 20, 255);
- }
-
- #[test]
- fn seek_cursor_end() {
- let mut cursor = get_u8_range();
-
- cursor.seek(SeekFrom::End(-10)).unwrap();
- check_read_all(cursor, 246, 255);
- }
-
- #[test]
- fn seek_cursor_error_too_long() {
- let mut cursor = get_u8_range();
- let res = cursor.seek(SeekFrom::Start(1000));
- let actual_error = res.expect_err("expected error").to_string();
- let expected_error =
- "Request out of bounds: cur position 0 + seek Start(1000) >= length 256: 1000";
- assert_eq!(actual_error, expected_error);
- }
-
- #[test]
- fn seek_cursor_error_too_short() {
- let mut cursor = get_u8_range();
- let res = cursor.seek(SeekFrom::End(-1000));
- let actual_error = res.expect_err("expected error").to_string();
- let expected_error =
- "Request out of bounds: cur position 0 + seek End(-1000) < 0: -744";
- assert_eq!(actual_error, expected_error);
- }
-}
diff --git a/parquet/src/util/hash_util.rs b/parquet/src/util/hash_util.rs
deleted file mode 100644
index f7849da..0000000
--- a/parquet/src/util/hash_util.rs
+++ /dev/null
@@ -1,172 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-use crate::data_type::AsBytes;
-
-/// Computes hash value for `data`, with a seed value `seed`.
-/// The data type `T` must implement the `AsBytes` trait.
-pub fn hash<T: AsBytes>(data: &T, seed: u32) -> u32 {
- hash_(data.as_bytes(), seed)
-}
-
-fn hash_(data: &[u8], seed: u32) -> u32 {
- #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
- unsafe {
- if is_x86_feature_detected!("sse4.2") {
- crc32_hash(data, seed)
- } else {
- murmur_hash2_64a(data, seed as u64) as u32
- }
- }
-
- #[cfg(any(target_arch = "aarch64", target_arch = "arm"))]
- unsafe {
- murmur_hash2_64a(data, seed as u64) as u32
- }
-}
-
-const MURMUR_PRIME: u64 = 0xc6a4a7935bd1e995;
-const MURMUR_R: i32 = 47;
-
-/// Rust implementation of MurmurHash2, 64-bit version for 64-bit platforms
-///
-/// SAFTETY Only safe on platforms which support unaligned loads (like x86_64)
-unsafe fn murmur_hash2_64a(data_bytes: &[u8], seed: u64) -> u64 {
- let len = data_bytes.len();
- let len_64 = (len / 8) * 8;
- let data_bytes_64 = std::slice::from_raw_parts(
- &data_bytes[0..len_64] as *const [u8] as *const u64,
- len / 8,
- );
-
- let mut h = seed ^ (MURMUR_PRIME.wrapping_mul(data_bytes.len() as u64));
- for v in data_bytes_64 {
- let mut k = *v;
- k = k.wrapping_mul(MURMUR_PRIME);
- k ^= k >> MURMUR_R;
- k = k.wrapping_mul(MURMUR_PRIME);
- h ^= k;
- h = h.wrapping_mul(MURMUR_PRIME);
- }
-
- let data2 = &data_bytes[len_64..];
-
- let v = len & 7;
- if v == 7 {
- h ^= (data2[6] as u64) << 48;
- }
- if v >= 6 {
- h ^= (data2[5] as u64) << 40;
- }
- if v >= 5 {
- h ^= (data2[4] as u64) << 32;
- }
- if v >= 4 {
- h ^= (data2[3] as u64) << 24;
- }
- if v >= 3 {
- h ^= (data2[2] as u64) << 16;
- }
- if v >= 2 {
- h ^= (data2[1] as u64) << 8;
- }
- if v >= 1 {
- h ^= data2[0] as u64;
- }
- if v > 0 {
- h = h.wrapping_mul(MURMUR_PRIME);
- }
-
- h ^= h >> MURMUR_R;
- h = h.wrapping_mul(MURMUR_PRIME);
- h ^= h >> MURMUR_R;
- h
-}
-
-/// CRC32 hash implementation using SSE4 instructions. Borrowed from Impala.
-#[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
-#[target_feature(enable = "sse4.2")]
-unsafe fn crc32_hash(bytes: &[u8], seed: u32) -> u32 {
- #[cfg(target_arch = "x86")]
- use std::arch::x86::*;
- #[cfg(target_arch = "x86_64")]
- use std::arch::x86_64::*;
-
- let u32_num_bytes = std::mem::size_of::<u32>();
- let mut num_bytes = bytes.len();
- let num_words = num_bytes / u32_num_bytes;
- num_bytes %= u32_num_bytes;
-
- let bytes_u32: &[u32] = std::slice::from_raw_parts(
- &bytes[0..num_words * u32_num_bytes] as *const [u8] as *const u32,
- num_words,
- );
-
- let mut offset = 0;
- let mut hash = seed;
- while offset < num_words {
- hash = _mm_crc32_u32(hash, bytes_u32[offset]);
- offset += 1;
- }
-
- offset = num_words * u32_num_bytes;
- while offset < num_bytes {
- hash = _mm_crc32_u8(hash, bytes[offset]);
- offset += 1;
- }
-
- // The lower half of the CRC hash has poor uniformity, so swap the halves
- // for anyone who only uses the first several bits of the hash.
- hash = (hash << 16) | (hash >> 16);
- hash
-}
-
-#[cfg(test)]
-mod tests {
- use super::*;
-
- #[test]
- fn test_murmur2_64a() {
- unsafe {
- let result = murmur_hash2_64a(b"hello", 123);
- assert_eq!(result, 2597646618390559622);
-
- let result = murmur_hash2_64a(b"helloworld", 123);
- assert_eq!(result, 4934371746140206573);
-
- let result = murmur_hash2_64a(b"helloworldparquet", 123);
- assert_eq!(result, 2392198230801491746);
- }
- }
-
- #[test]
- #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
- fn test_crc32() {
- if is_x86_feature_detected!("sse4.2") {
- unsafe {
- let result = crc32_hash(b"hello", 123);
- assert_eq!(result, 2927487359);
-
- let result = crc32_hash(b"helloworld", 123);
- assert_eq!(result, 314229527);
-
- let result = crc32_hash(b"helloworldparquet", 123);
- assert_eq!(result, 667078870);
- }
- }
- }
-}
diff --git a/parquet/src/util/io.rs b/parquet/src/util/io.rs
deleted file mode 100644
index 44e99ac..0000000
--- a/parquet/src/util/io.rs
+++ /dev/null
@@ -1,329 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-use std::{cell::RefCell, cmp, fmt, io::*};
-
-use crate::file::{reader::Length, writer::ParquetWriter};
-
-const DEFAULT_BUF_SIZE: usize = 8 * 1024;
-
-// ----------------------------------------------------------------------
-
-/// TryClone tries to clone the type and should maintain the `Seek` position of the given
-/// instance.
-pub trait TryClone: Sized {
- /// Clones the type returning a new instance or an error if it's not possible
- /// to clone it.
- fn try_clone(&self) -> Result<Self>;
-}
-
-/// ParquetReader is the interface which needs to be fulfilled to be able to parse a
-/// parquet source.
-pub trait ParquetReader: Read + Seek + Length + TryClone {}
-impl<T: Read + Seek + Length + TryClone> ParquetReader for T {}
-
-// Read/Write wrappers for `File`.
-
-/// Position trait returns the current position in the stream.
-/// Should be viewed as a lighter version of `Seek` that does not allow seek operations,
-/// and does not require mutable reference for the current position.
-pub trait Position {
- /// Returns position in the stream.
- fn pos(&self) -> u64;
-}
-
-/// Struct that represents a slice of a file data with independent start position and
-/// length. Internally clones provided file handle, wraps with a custom implementation
-/// of BufReader that resets position before any read.
-///
-/// This is workaround and alternative for `file.try_clone()` method. It clones `File`
-/// while preserving independent position, which is not available with `try_clone()`.
-///
-/// Designed after `arrow::io::RandomAccessFile` and `std::io::BufReader`
-pub struct FileSource<R: ParquetReader> {
- reader: RefCell<R>,
- start: u64, // start position in a file
- end: u64, // end position in a file
- buf: Vec<u8>, // buffer where bytes read in advance are stored
- buf_pos: usize, // current position of the reader in the buffer
- buf_cap: usize, // current number of bytes read into the buffer
-}
-
-impl<R: ParquetReader> fmt::Debug for FileSource<R> {
- fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
- f.debug_struct("FileSource")
- .field("reader", &"OPAQUE")
- .field("start", &self.start)
- .field("end", &self.end)
- .field("buf.len", &self.buf.len())
- .field("buf_pos", &self.buf_pos)
- .field("buf_cap", &self.buf_cap)
- .finish()
- }
-}
-
-impl<R: ParquetReader> FileSource<R> {
- /// Creates new file reader with start and length from a file handle
- pub fn new(fd: &R, start: u64, length: usize) -> Self {
- let reader = RefCell::new(fd.try_clone().unwrap());
- Self {
- reader,
- start,
- end: start + length as u64,
- buf: vec![0_u8; DEFAULT_BUF_SIZE],
- buf_pos: 0,
- buf_cap: 0,
- }
- }
-
- fn fill_inner_buf(&mut self) -> Result<&[u8]> {
- if self.buf_pos >= self.buf_cap {
- // If we've reached the end of our internal buffer then we need to fetch
- // some more data from the underlying reader.
- // Branch using `>=` instead of the more correct `==`
- // to tell the compiler that the pos..cap slice is always valid.
- debug_assert!(self.buf_pos == self.buf_cap);
- let mut reader = self.reader.borrow_mut();
- reader.seek(SeekFrom::Start(self.start))?; // always seek to start before reading
- self.buf_cap = reader.read(&mut self.buf)?;
- self.buf_pos = 0;
- }
- Ok(&self.buf[self.buf_pos..self.buf_cap])
- }
-
- fn skip_inner_buf(&mut self, buf: &mut [u8]) -> Result<usize> {
- // discard buffer
- self.buf_pos = 0;
- self.buf_cap = 0;
- // read directly into param buffer
- let mut reader = self.reader.borrow_mut();
- reader.seek(SeekFrom::Start(self.start))?; // always seek to start before reading
- let nread = reader.read(buf)?;
- self.start += nread as u64;
- Ok(nread)
- }
-}
-
-impl<R: ParquetReader> Read for FileSource<R> {
- fn read(&mut self, buf: &mut [u8]) -> Result<usize> {
- let bytes_to_read = cmp::min(buf.len(), (self.end - self.start) as usize);
- let buf = &mut buf[0..bytes_to_read];
-
- // If we don't have any buffered data and we're doing a massive read
- // (larger than our internal buffer), bypass our internal buffer
- // entirely.
- if self.buf_pos == self.buf_cap && buf.len() >= self.buf.len() {
- return self.skip_inner_buf(buf);
- }
- let nread = {
- let mut rem = self.fill_inner_buf()?;
- // copy the data from the inner buffer to the param buffer
- rem.read(buf)?
- };
- // consume from buffer
- self.buf_pos = cmp::min(self.buf_pos + nread, self.buf_cap);
-
- self.start += nread as u64;
- Ok(nread)
- }
-}
-
-impl<R: ParquetReader> Position for FileSource<R> {
- fn pos(&self) -> u64 {
- self.start
- }
-}
-
-impl<R: ParquetReader> Length for FileSource<R> {
- fn len(&self) -> u64 {
- self.end - self.start
- }
-}
-
-/// Struct that represents `File` output stream with position tracking.
-/// Used as a sink in file writer.
-pub struct FileSink<W: ParquetWriter> {
- buf: BufWriter<W>,
- // This is not necessarily position in the underlying file,
- // but rather current position in the sink.
- pos: u64,
-}
-
-impl<W: ParquetWriter> FileSink<W> {
- /// Creates new file sink.
- /// Position is set to whatever position file has.
- pub fn new(buf: &W) -> Self {
- let mut owned_buf = buf.try_clone().unwrap();
- let pos = owned_buf.seek(SeekFrom::Current(0)).unwrap();
- Self {
- buf: BufWriter::new(owned_buf),
- pos,
- }
- }
-}
-
-impl<W: ParquetWriter> Write for FileSink<W> {
- fn write(&mut self, buf: &[u8]) -> Result<usize> {
- let num_bytes = self.buf.write(buf)?;
- self.pos += num_bytes as u64;
- Ok(num_bytes)
- }
-
- fn flush(&mut self) -> Result<()> {
- self.buf.flush()
- }
-}
-
-impl<W: ParquetWriter> Position for FileSink<W> {
- fn pos(&self) -> u64 {
- self.pos
- }
-}
-
-// Position implementation for Cursor to use in various tests.
-impl<'a> Position for Cursor<&'a mut Vec<u8>> {
- fn pos(&self) -> u64 {
- self.position()
- }
-}
-
-#[cfg(test)]
-mod tests {
- use super::*;
-
- use std::iter;
-
- use crate::util::test_common::{get_temp_file, get_test_file};
-
- #[test]
- fn test_io_read_fully() {
- let mut buf = vec![0; 8];
- let mut src = FileSource::new(&get_test_file("alltypes_plain.parquet"), 0, 4);
-
- let bytes_read = src.read(&mut buf[..]).unwrap();
- assert_eq!(bytes_read, 4);
- assert_eq!(buf, vec![b'P', b'A', b'R', b'1', 0, 0, 0, 0]);
- }
-
- #[test]
- fn test_io_read_in_chunks() {
- let mut buf = vec![0; 4];
- let mut src = FileSource::new(&get_test_file("alltypes_plain.parquet"), 0, 4);
-
- let bytes_read = src.read(&mut buf[0..2]).unwrap();
- assert_eq!(bytes_read, 2);
- let bytes_read = src.read(&mut buf[2..]).unwrap();
- assert_eq!(bytes_read, 2);
- assert_eq!(buf, vec![b'P', b'A', b'R', b'1']);
- }
-
- #[test]
- fn test_io_read_pos() {
- let mut src = FileSource::new(&get_test_file("alltypes_plain.parquet"), 0, 4);
-
- let _ = src.read(&mut [0; 1]).unwrap();
- assert_eq!(src.pos(), 1);
-
- let _ = src.read(&mut [0; 4]).unwrap();
- assert_eq!(src.pos(), 4);
- }
-
- #[test]
- fn test_io_read_over_limit() {
- let mut src = FileSource::new(&get_test_file("alltypes_plain.parquet"), 0, 4);
-
- // Read all bytes from source
- let _ = src.read(&mut [0; 128]).unwrap();
- assert_eq!(src.pos(), 4);
-
- // Try reading again, should return 0 bytes.
- let bytes_read = src.read(&mut [0; 128]).unwrap();
- assert_eq!(bytes_read, 0);
- assert_eq!(src.pos(), 4);
- }
-
- #[test]
- fn test_io_seek_switch() {
- let mut buf = vec![0; 4];
- let mut file = get_test_file("alltypes_plain.parquet");
- let mut src = FileSource::new(&file, 0, 4);
-
- file.seek(SeekFrom::Start(5_u64))
- .expect("File seek to a position");
-
- let bytes_read = src.read(&mut buf[..]).unwrap();
- assert_eq!(bytes_read, 4);
- assert_eq!(buf, vec![b'P', b'A', b'R', b'1']);
- }
-
- #[test]
- fn test_io_write_with_pos() {
- let mut file = get_temp_file("file_sink_test", &[b'a', b'b', b'c']);
- file.seek(SeekFrom::Current(3)).unwrap();
-
- // Write into sink
- let mut sink = FileSink::new(&file);
- assert_eq!(sink.pos(), 3);
-
- sink.write_all(&[b'd', b'e', b'f', b'g']).unwrap();
- assert_eq!(sink.pos(), 7);
-
- sink.flush().unwrap();
- assert_eq!(sink.pos(), file.seek(SeekFrom::Current(0)).unwrap());
-
- // Read data using file chunk
- let mut res = vec![0u8; 7];
- let mut chunk =
- FileSource::new(&file, 0, file.metadata().unwrap().len() as usize);
- chunk.read_exact(&mut res[..]).unwrap();
- assert_eq!(res, vec![b'a', b'b', b'c', b'd', b'e', b'f', b'g']);
- }
-
- #[test]
- fn test_io_large_read() {
- // Generate repeated 'abcdef' pattern and write it into a file
- let patterned_data: Vec<u8> = iter::repeat(vec![0, 1, 2, 3, 4, 5])
- .flatten()
- .take(3 * DEFAULT_BUF_SIZE)
- .collect();
- // always use different temp files as test might be run in parallel
- let mut file = get_temp_file("large_file_sink_test", &patterned_data);
-
- // seek the underlying file to the first 'd'
- file.seek(SeekFrom::Start(3)).unwrap();
-
- // create the FileSource reader that starts at pos 1 ('b')
- let mut chunk = FileSource::new(&file, 1, patterned_data.len() - 1);
-
- // read the 'b' at pos 1
- let mut res = vec![0u8; 1];
- chunk.read_exact(&mut res).unwrap();
- assert_eq!(res, &[1]);
-
- // the underlying file is sought to 'e'
- file.seek(SeekFrom::Start(4)).unwrap();
-
- // now read large chunk that starts with 'c' (after 'b')
- let mut res = vec![0u8; 2 * DEFAULT_BUF_SIZE];
- chunk.read_exact(&mut res).unwrap();
- assert_eq!(
- res,
- &patterned_data[2..2 + 2 * DEFAULT_BUF_SIZE],
- "read buf and original data are not equal"
- );
- }
-}
diff --git a/parquet/src/util/memory.rs b/parquet/src/util/memory.rs
deleted file mode 100644
index 1642a4b..0000000
--- a/parquet/src/util/memory.rs
+++ /dev/null
@@ -1,545 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-//! Utility methods and structs for working with memory.
-
-use std::{
- fmt::{Debug, Display, Formatter, Result as FmtResult},
- io::{Result as IoResult, Write},
- mem,
- ops::{Index, IndexMut},
- sync::{
- atomic::{AtomicI64, Ordering},
- Arc, Weak,
- },
-};
-
-// ----------------------------------------------------------------------
-// Memory Tracker classes
-
-/// Reference counted pointer for [`MemTracker`].
-pub type MemTrackerPtr = Arc<MemTracker>;
-/// Non-owning reference for [`MemTracker`].
-pub type WeakMemTrackerPtr = Weak<MemTracker>;
-
-/// Struct to track memory usage information.
-#[derive(Debug)]
-pub struct MemTracker {
- // In the tuple, the first element is the current memory allocated (in bytes),
- // and the second element is the maximum memory allocated so far (in bytes).
- current_memory_usage: AtomicI64,
- max_memory_usage: AtomicI64,
-}
-
-impl MemTracker {
- /// Creates new memory tracker.
- #[inline]
- pub fn new() -> MemTracker {
- MemTracker {
- current_memory_usage: Default::default(),
- max_memory_usage: Default::default(),
- }
- }
-
- /// Returns the current memory consumption, in bytes.
- pub fn memory_usage(&self) -> i64 {
- self.current_memory_usage.load(Ordering::Acquire)
- }
-
- /// Returns the maximum memory consumption so far, in bytes.
- pub fn max_memory_usage(&self) -> i64 {
- self.max_memory_usage.load(Ordering::Acquire)
- }
-
- /// Adds `num_bytes` to the memory consumption tracked by this memory tracker.
- #[inline]
- pub fn alloc(&self, num_bytes: i64) {
- let new_current = self
- .current_memory_usage
- .fetch_add(num_bytes, Ordering::Acquire)
- + num_bytes;
- self.max_memory_usage
- .fetch_max(new_current, Ordering::Acquire);
- }
-}
-
-// ----------------------------------------------------------------------
-// Buffer classes
-
-/// Type alias for [`Buffer`].
-pub type ByteBuffer = Buffer<u8>;
-/// Type alias for [`BufferPtr`].
-pub type ByteBufferPtr = BufferPtr<u8>;
-
-/// A resize-able buffer class with generic member, with optional memory tracker.
-///
-/// Note that a buffer has two attributes:
-/// `capacity` and `size`: the former is the total number of space reserved for
-/// the buffer, while the latter is the actual number of elements.
-/// Invariant: `capacity` >= `size`.
-/// The total allocated bytes for a buffer equals to `capacity * sizeof<T>()`.
-pub struct Buffer<T: Clone> {
- data: Vec<T>,
- mem_tracker: Option<MemTrackerPtr>,
- type_length: usize,
-}
-
-impl<T: Clone> Buffer<T> {
- /// Creates new empty buffer.
- pub fn new() -> Self {
- Buffer {
- data: vec![],
- mem_tracker: None,
- type_length: std::mem::size_of::<T>(),
- }
- }
-
- /// Adds [`MemTracker`] for this buffer.
- #[inline]
- pub fn with_mem_tracker(mut self, mc: MemTrackerPtr) -> Self {
- mc.alloc((self.data.capacity() * self.type_length) as i64);
- self.mem_tracker = Some(mc);
- self
- }
-
- /// Returns slice of data in this buffer.
- #[inline]
- pub fn data(&self) -> &[T] {
- self.data.as_slice()
- }
-
- /// Sets data for this buffer.
- #[inline]
- pub fn set_data(&mut self, new_data: Vec<T>) {
- if let Some(ref mc) = self.mem_tracker {
- let capacity_diff = new_data.capacity() as i64 - self.data.capacity() as i64;
- mc.alloc(capacity_diff * self.type_length as i64);
- }
- self.data = new_data;
- }
-
- /// Resizes underlying data in place to a new length `new_size`.
- ///
- /// If `new_size` is less than current length, data is truncated, otherwise, it is
- /// extended to `new_size` with provided default value `init_value`.
- ///
- /// Memory tracker is also updated, if available.
- #[inline]
- pub fn resize(&mut self, new_size: usize, init_value: T) {
- let old_capacity = self.data.capacity();
- self.data.resize(new_size, init_value);
- if let Some(ref mc) = self.mem_tracker {
- let capacity_diff = self.data.capacity() as i64 - old_capacity as i64;
- mc.alloc(capacity_diff * self.type_length as i64);
- }
- }
-
- /// Clears underlying data.
- #[inline]
- pub fn clear(&mut self) {
- self.data.clear()
- }
-
- /// Reserves capacity `additional_capacity` for underlying data vector.
- ///
- /// Memory tracker is also updated, if available.
- #[inline]
- pub fn reserve(&mut self, additional_capacity: usize) {
- let old_capacity = self.data.capacity();
- self.data.reserve(additional_capacity);
- if self.data.capacity() > old_capacity {
- if let Some(ref mc) = self.mem_tracker {
- let capacity_diff = self.data.capacity() as i64 - old_capacity as i64;
- mc.alloc(capacity_diff * self.type_length as i64);
- }
- }
- }
-
- /// Returns [`BufferPtr`] with buffer data.
- /// Buffer data is reset.
- #[inline]
- pub fn consume(&mut self) -> BufferPtr<T> {
- let old_data = mem::replace(&mut self.data, vec![]);
- let mut result = BufferPtr::new(old_data);
- if let Some(ref mc) = self.mem_tracker {
- result = result.with_mem_tracker(mc.clone());
- }
- result
- }
-
- /// Adds `value` to the buffer.
- #[inline]
- pub fn push(&mut self, value: T) {
- self.data.push(value)
- }
-
- /// Returns current capacity for the buffer.
- #[inline]
- pub fn capacity(&self) -> usize {
- self.data.capacity()
- }
-
- /// Returns current size for the buffer.
- #[inline]
- pub fn size(&self) -> usize {
- self.data.len()
- }
-
- /// Returns `true` if memory tracker is added to buffer, `false` otherwise.
- #[inline]
- pub fn is_mem_tracked(&self) -> bool {
- self.mem_tracker.is_some()
- }
-
- /// Returns memory tracker associated with this buffer.
- /// This may panic, if memory tracker is not set, use method above to check if
- /// memory tracker is available.
- #[inline]
- pub fn mem_tracker(&self) -> &MemTrackerPtr {
- self.mem_tracker.as_ref().unwrap()
- }
-}
-
-impl<T: Sized + Clone> Index<usize> for Buffer<T> {
- type Output = T;
-
- fn index(&self, index: usize) -> &T {
- &self.data[index]
- }
-}
-
-impl<T: Sized + Clone> IndexMut<usize> for Buffer<T> {
- fn index_mut(&mut self, index: usize) -> &mut T {
- &mut self.data[index]
- }
-}
-
-// TODO: implement this for other types
-impl Write for Buffer<u8> {
- #[inline]
- fn write(&mut self, buf: &[u8]) -> IoResult<usize> {
- let old_capacity = self.data.capacity();
- let bytes_written = self.data.write(buf)?;
- if let Some(ref mc) = self.mem_tracker {
- if self.data.capacity() - old_capacity > 0 {
- mc.alloc((self.data.capacity() - old_capacity) as i64)
- }
- }
- Ok(bytes_written)
- }
-
- fn flush(&mut self) -> IoResult<()> {
- // No-op
- self.data.flush()
- }
-}
-
-impl AsRef<[u8]> for Buffer<u8> {
- fn as_ref(&self) -> &[u8] {
- self.data.as_slice()
- }
-}
-
-impl<T: Clone> Drop for Buffer<T> {
- #[inline]
- fn drop(&mut self) {
- if let Some(ref mc) = self.mem_tracker {
- mc.alloc(-((self.data.capacity() * self.type_length) as i64));
- }
- }
-}
-
-// ----------------------------------------------------------------------
-// Immutable Buffer (BufferPtr) classes
-
-/// An representation of a slice on a reference-counting and read-only byte array.
-/// Sub-slices can be further created from this. The byte array will be released
-/// when all slices are dropped.
-#[allow(clippy::rc_buffer)]
-#[derive(Clone, Debug)]
-pub struct BufferPtr<T> {
- data: Arc<Vec<T>>,
- start: usize,
- len: usize,
- // TODO: will this create too many references? rethink about this.
- mem_tracker: Option<MemTrackerPtr>,
-}
-
-impl<T> BufferPtr<T> {
- /// Creates new buffer from a vector.
- pub fn new(v: Vec<T>) -> Self {
- let len = v.len();
- Self {
- data: Arc::new(v),
- start: 0,
- len,
- mem_tracker: None,
- }
- }
-
- /// Returns slice of data in this buffer.
- #[inline]
- pub fn data(&self) -> &[T] {
- &self.data[self.start..self.start + self.len]
- }
-
- /// Updates this buffer with new `start` position and length `len`.
- ///
- /// Range should be within current start position and length.
- #[inline]
- pub fn with_range(mut self, start: usize, len: usize) -> Self {
- self.set_range(start, len);
- self
- }
-
- /// Updates this buffer with new `start` position and length `len`.
- ///
- /// Range should be within current start position and length.
- #[inline]
- pub fn set_range(&mut self, start: usize, len: usize) {
- assert!(self.start <= start && start + len <= self.start + self.len);
- self.start = start;
- self.len = len;
- }
-
- /// Adds memory tracker to this buffer.
- pub fn with_mem_tracker(mut self, mc: MemTrackerPtr) -> Self {
- self.mem_tracker = Some(mc);
- self
- }
-
- /// Returns start position of this buffer.
- #[inline]
- pub fn start(&self) -> usize {
- self.start
- }
-
- /// Returns length of this buffer
- #[inline]
- pub fn len(&self) -> usize {
- self.len
- }
-
- /// Returns whether this buffer is empty
- #[inline]
- pub fn is_empty(&self) -> bool {
- self.len == 0
- }
-
- /// Returns `true` if this buffer has memory tracker, `false` otherwise.
- pub fn is_mem_tracked(&self) -> bool {
- self.mem_tracker.is_some()
- }
-
- /// Returns a shallow copy of the buffer.
- /// Reference counted pointer to the data is copied.
- pub fn all(&self) -> BufferPtr<T> {
- BufferPtr {
- data: self.data.clone(),
- start: self.start,
- len: self.len,
- mem_tracker: self.mem_tracker.as_ref().cloned(),
- }
- }
-
- /// Returns a shallow copy of the buffer that starts with `start` position.
- pub fn start_from(&self, start: usize) -> BufferPtr<T> {
- assert!(start <= self.len);
- BufferPtr {
- data: self.data.clone(),
- start: self.start + start,
- len: self.len - start,
- mem_tracker: self.mem_tracker.as_ref().cloned(),
- }
- }
-
- /// Returns a shallow copy that is a range slice within this buffer.
- pub fn range(&self, start: usize, len: usize) -> BufferPtr<T> {
- assert!(start + len <= self.len);
- BufferPtr {
- data: self.data.clone(),
- start: self.start + start,
- len,
- mem_tracker: self.mem_tracker.as_ref().cloned(),
- }
- }
-}
-
-impl<T: Sized> Index<usize> for BufferPtr<T> {
- type Output = T;
-
- fn index(&self, index: usize) -> &T {
- assert!(index < self.len);
- &self.data[self.start + index]
- }
-}
-
-impl<T: Debug> Display for BufferPtr<T> {
- fn fmt(&self, f: &mut Formatter) -> FmtResult {
- write!(f, "{:?}", self.data)
- }
-}
-
-impl<T> Drop for BufferPtr<T> {
- fn drop(&mut self) {
- if let Some(ref mc) = self.mem_tracker {
- if Arc::strong_count(&self.data) == 1 && Arc::weak_count(&self.data) == 0 {
- mc.alloc(-(self.data.capacity() as i64));
- }
- }
- }
-}
-
-impl AsRef<[u8]> for BufferPtr<u8> {
- #[inline]
- fn as_ref(&self) -> &[u8] {
- &self.data[self.start..self.start + self.len]
- }
-}
-
-#[cfg(test)]
-mod tests {
- use super::*;
-
- #[test]
- fn test_byte_buffer_mem_tracker() {
- let mem_tracker = Arc::new(MemTracker::new());
-
- let mut buffer = ByteBuffer::new().with_mem_tracker(mem_tracker.clone());
- buffer.set_data(vec![0; 10]);
- assert_eq!(mem_tracker.memory_usage(), buffer.capacity() as i64);
- buffer.set_data(vec![0; 20]);
- let capacity = buffer.capacity() as i64;
- assert_eq!(mem_tracker.memory_usage(), capacity);
-
- let max_capacity = {
- let mut buffer2 = ByteBuffer::new().with_mem_tracker(mem_tracker.clone());
- buffer2.reserve(30);
- assert_eq!(
- mem_tracker.memory_usage(),
- buffer2.capacity() as i64 + capacity
- );
- buffer2.set_data(vec![0; 100]);
- assert_eq!(
- mem_tracker.memory_usage(),
- buffer2.capacity() as i64 + capacity
- );
- buffer2.capacity() as i64 + capacity
- };
-
- assert_eq!(mem_tracker.memory_usage(), capacity);
- assert_eq!(mem_tracker.max_memory_usage(), max_capacity);
-
- buffer.reserve(40);
- assert_eq!(mem_tracker.memory_usage(), buffer.capacity() as i64);
-
- buffer.consume();
- assert_eq!(mem_tracker.memory_usage(), buffer.capacity() as i64);
- }
-
- #[test]
- fn test_byte_ptr_mem_tracker() {
- let mem_tracker = Arc::new(MemTracker::new());
-
- let mut buffer = ByteBuffer::new().with_mem_tracker(mem_tracker.clone());
- buffer.set_data(vec![0; 60]);
-
- {
- let buffer_capacity = buffer.capacity() as i64;
- let buf_ptr = buffer.consume();
- assert_eq!(mem_tracker.memory_usage(), buffer_capacity);
- {
- let buf_ptr1 = buf_ptr.all();
- {
- let _ = buf_ptr.start_from(20);
- assert_eq!(mem_tracker.memory_usage(), buffer_capacity);
- }
- assert_eq!(mem_tracker.memory_usage(), buffer_capacity);
- let _ = buf_ptr1.range(30, 20);
- assert_eq!(mem_tracker.memory_usage(), buffer_capacity);
- }
- assert_eq!(mem_tracker.memory_usage(), buffer_capacity);
- }
- assert_eq!(mem_tracker.memory_usage(), buffer.capacity() as i64);
- }
-
- #[test]
- fn test_byte_buffer() {
- let mut buffer = ByteBuffer::new();
- assert_eq!(buffer.size(), 0);
- assert_eq!(buffer.capacity(), 0);
-
- let mut buffer2 = ByteBuffer::new();
- buffer2.reserve(40);
- assert_eq!(buffer2.size(), 0);
- assert_eq!(buffer2.capacity(), 40);
-
- buffer.set_data((0..5).collect());
- assert_eq!(buffer.size(), 5);
- assert_eq!(buffer[4], 4);
-
- buffer.set_data((0..20).collect());
- assert_eq!(buffer.size(), 20);
- assert_eq!(buffer[10], 10);
-
- let expected: Vec<u8> = (0..20).collect();
- {
- let data = buffer.data();
- assert_eq!(data, expected.as_slice());
- }
-
- buffer.reserve(40);
- assert!(buffer.capacity() >= 40);
-
- let byte_ptr = buffer.consume();
- assert_eq!(buffer.size(), 0);
- assert_eq!(byte_ptr.as_ref(), expected.as_slice());
-
- let values: Vec<u8> = (0..30).collect();
- let _ = buffer.write(values.as_slice());
- let _ = buffer.flush();
-
- assert_eq!(buffer.data(), values.as_slice());
- }
-
- #[test]
- fn test_byte_ptr() {
- let values = (0..50).collect();
- let ptr = ByteBufferPtr::new(values);
- assert_eq!(ptr.len(), 50);
- assert_eq!(ptr.start(), 0);
- assert_eq!(ptr[40], 40);
-
- let ptr2 = ptr.all();
- assert_eq!(ptr2.len(), 50);
- assert_eq!(ptr2.start(), 0);
- assert_eq!(ptr2[40], 40);
-
- let ptr3 = ptr.start_from(20);
- assert_eq!(ptr3.len(), 30);
- assert_eq!(ptr3.start(), 20);
- assert_eq!(ptr3[0], 20);
-
- let ptr4 = ptr3.range(10, 10);
- assert_eq!(ptr4.len(), 10);
- assert_eq!(ptr4.start(), 30);
- assert_eq!(ptr4[0], 30);
-
- let expected: Vec<u8> = (30..40).collect();
- assert_eq!(ptr4.as_ref(), expected.as_slice());
- }
-}
diff --git a/parquet/src/util/mod.rs b/parquet/src/util/mod.rs
deleted file mode 100644
index 8f6d85d..0000000
--- a/parquet/src/util/mod.rs
+++ /dev/null
@@ -1,28 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-pub mod io;
-pub mod memory;
-#[macro_use]
-pub mod bit_util;
-mod bit_packing;
-pub mod cursor;
-pub mod hash_util;
-pub(crate) mod test_common;
-pub use self::test_common::page_util::{
- DataPageBuilder, DataPageBuilderImpl, InMemoryPageIterator,
-};
diff --git a/parquet/src/util/test_common/file_util.rs b/parquet/src/util/test_common/file_util.rs
deleted file mode 100644
index 7393b55..0000000
--- a/parquet/src/util/test_common/file_util.rs
+++ /dev/null
@@ -1,73 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-use std::{env, fs, io::Write, path::PathBuf, str::FromStr};
-
-/// Returns path to the test parquet file in 'data' directory
-pub fn get_test_path(file_name: &str) -> PathBuf {
- let mut pathbuf =
- PathBuf::from_str(&arrow::util::test_util::parquet_test_data()).unwrap();
- pathbuf.push(file_name);
- pathbuf
-}
-
-/// Returns file handle for a test parquet file from 'data' directory
-pub fn get_test_file(file_name: &str) -> fs::File {
- let path = get_test_path(file_name);
- fs::File::open(path.as_path()).unwrap_or_else(|err| {
- panic!(
- "Test file {} could not be opened, did you do `git submodule update`?: {}",
- path.display(),
- err
- )
- })
-}
-
-/// Returns file handle for a temp file in 'target' directory with a provided content
-pub fn get_temp_file(file_name: &str, content: &[u8]) -> fs::File {
- // build tmp path to a file in "target/debug/testdata"
- let mut path_buf = env::current_dir().unwrap();
- path_buf.push("target");
- path_buf.push("debug");
- path_buf.push("testdata");
- fs::create_dir_all(&path_buf).unwrap();
- path_buf.push(file_name);
-
- // write file content
- let mut tmp_file = fs::File::create(path_buf.as_path()).unwrap();
- tmp_file.write_all(content).unwrap();
- tmp_file.sync_all().unwrap();
-
- // return file handle for both read and write
- let file = fs::OpenOptions::new()
- .read(true)
- .write(true)
- .open(path_buf.as_path());
- assert!(file.is_ok());
- file.unwrap()
-}
-
-pub fn get_temp_filename() -> PathBuf {
- let mut path_buf = env::current_dir().unwrap();
- path_buf.push("target");
- path_buf.push("debug");
- path_buf.push("testdata");
- fs::create_dir_all(&path_buf).unwrap();
- path_buf.push(rand::random::<i16>().to_string());
-
- path_buf
-}
diff --git a/parquet/src/util/test_common/mod.rs b/parquet/src/util/test_common/mod.rs
deleted file mode 100644
index ed65bbe..0000000
--- a/parquet/src/util/test_common/mod.rs
+++ /dev/null
@@ -1,33 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-pub mod file_util;
-pub mod page_util;
-pub mod rand_gen;
-
-pub use self::rand_gen::random_bools;
-pub use self::rand_gen::random_bytes;
-pub use self::rand_gen::random_numbers;
-pub use self::rand_gen::random_numbers_range;
-pub use self::rand_gen::RandGen;
-
-pub use self::file_util::get_temp_file;
-pub use self::file_util::get_temp_filename;
-pub use self::file_util::get_test_file;
-pub use self::file_util::get_test_path;
-
-pub use self::page_util::make_pages;
diff --git a/parquet/src/util/test_common/page_util.rs b/parquet/src/util/test_common/page_util.rs
deleted file mode 100644
index 581845a..0000000
--- a/parquet/src/util/test_common/page_util.rs
+++ /dev/null
@@ -1,320 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-use crate::basic::Encoding;
-use crate::column::page::PageReader;
-use crate::column::page::{Page, PageIterator};
-use crate::data_type::DataType;
-use crate::encodings::encoding::{get_encoder, DictEncoder, Encoder};
-use crate::encodings::levels::max_buffer_size;
-use crate::encodings::levels::LevelEncoder;
-use crate::errors::Result;
-use crate::schema::types::{ColumnDescPtr, SchemaDescPtr};
-use crate::util::memory::ByteBufferPtr;
-use crate::util::memory::MemTracker;
-use crate::util::memory::MemTrackerPtr;
-use crate::util::test_common::random_numbers_range;
-use rand::distributions::uniform::SampleUniform;
-use std::collections::VecDeque;
-use std::mem;
-use std::sync::Arc;
-
-pub trait DataPageBuilder {
- fn add_rep_levels(&mut self, max_level: i16, rep_levels: &[i16]);
- fn add_def_levels(&mut self, max_level: i16, def_levels: &[i16]);
- fn add_values<T: DataType>(&mut self, encoding: Encoding, values: &[T::T]);
- fn add_indices(&mut self, indices: ByteBufferPtr);
- fn consume(self) -> Page;
-}
-
-/// A utility struct for building data pages (v1 or v2). Callers must call:
-/// - add_rep_levels()
-/// - add_def_levels()
-/// - add_values() for normal data page / add_indices() for dictionary data page
-/// - consume()
-/// in order to populate and obtain a data page.
-pub struct DataPageBuilderImpl {
- desc: ColumnDescPtr,
- encoding: Option<Encoding>,
- mem_tracker: MemTrackerPtr,
- num_values: u32,
- buffer: Vec<u8>,
- rep_levels_byte_len: u32,
- def_levels_byte_len: u32,
- datapage_v2: bool,
-}
-
-impl DataPageBuilderImpl {
- // `num_values` is the number of non-null values to put in the data page.
- // `datapage_v2` flag is used to indicate if the generated data page should use V2
- // format or not.
- pub fn new(desc: ColumnDescPtr, num_values: u32, datapage_v2: bool) -> Self {
- DataPageBuilderImpl {
- desc,
- encoding: None,
- mem_tracker: Arc::new(MemTracker::new()),
- num_values,
- buffer: vec![],
- rep_levels_byte_len: 0,
- def_levels_byte_len: 0,
- datapage_v2,
- }
- }
-
- // Adds levels to the buffer and return number of encoded bytes
- fn add_levels(&mut self, max_level: i16, levels: &[i16]) -> u32 {
- if max_level <= 0 {
- return 0;
- }
- let size = max_buffer_size(Encoding::RLE, max_level, levels.len());
- let mut level_encoder = LevelEncoder::v1(Encoding::RLE, max_level, vec![0; size]);
- level_encoder.put(levels).expect("put() should be OK");
- let encoded_levels = level_encoder.consume().expect("consume() should be OK");
- // Actual encoded bytes (without length offset)
- let encoded_bytes = &encoded_levels[mem::size_of::<i32>()..];
- if self.datapage_v2 {
- // Level encoder always initializes with offset of i32, where it stores
- // length of encoded data; for data page v2 we explicitly
- // store length, therefore we should skip i32 bytes.
- self.buffer.extend_from_slice(encoded_bytes);
- } else {
- self.buffer.extend_from_slice(encoded_levels.as_slice());
- }
- encoded_bytes.len() as u32
- }
-}
-
-impl DataPageBuilder for DataPageBuilderImpl {
- fn add_rep_levels(&mut self, max_levels: i16, rep_levels: &[i16]) {
- self.num_values = rep_levels.len() as u32;
- self.rep_levels_byte_len = self.add_levels(max_levels, rep_levels);
- }
-
- fn add_def_levels(&mut self, max_levels: i16, def_levels: &[i16]) {
- assert!(
- self.num_values == def_levels.len() as u32,
- "Must call `add_rep_levels() first!`"
- );
-
- self.def_levels_byte_len = self.add_levels(max_levels, def_levels);
- }
-
- fn add_values<T: DataType>(&mut self, encoding: Encoding, values: &[T::T]) {
- assert!(
- self.num_values >= values.len() as u32,
- "num_values: {}, values.len(): {}",
- self.num_values,
- values.len()
- );
- self.encoding = Some(encoding);
- let mut encoder: Box<dyn Encoder<T>> =
- get_encoder::<T>(self.desc.clone(), encoding, self.mem_tracker.clone())
- .expect("get_encoder() should be OK");
- encoder.put(values).expect("put() should be OK");
- let encoded_values = encoder
- .flush_buffer()
- .expect("consume_buffer() should be OK");
- self.buffer.extend_from_slice(encoded_values.data());
- }
-
- fn add_indices(&mut self, indices: ByteBufferPtr) {
- self.encoding = Some(Encoding::RLE_DICTIONARY);
- self.buffer.extend_from_slice(indices.data());
- }
-
- fn consume(self) -> Page {
- if self.datapage_v2 {
- Page::DataPageV2 {
- buf: ByteBufferPtr::new(self.buffer),
- num_values: self.num_values,
- encoding: self.encoding.unwrap(),
- num_nulls: 0, /* set to dummy value - don't need this when reading
- * data page */
- num_rows: self.num_values, /* also don't need this when reading
- * data page */
- def_levels_byte_len: self.def_levels_byte_len,
- rep_levels_byte_len: self.rep_levels_byte_len,
- is_compressed: false,
- statistics: None, // set to None, we do not need statistics for tests
- }
- } else {
- Page::DataPage {
- buf: ByteBufferPtr::new(self.buffer),
- num_values: self.num_values,
- encoding: self.encoding.unwrap(),
- def_level_encoding: Encoding::RLE,
- rep_level_encoding: Encoding::RLE,
- statistics: None, // set to None, we do not need statistics for tests
- }
- }
- }
-}
-
-/// A utility page reader which stores pages in memory.
-pub struct InMemoryPageReader<P: Iterator<Item = Page>> {
- page_iter: P,
-}
-
-impl<P: Iterator<Item = Page>> InMemoryPageReader<P> {
- pub fn new(pages: impl IntoIterator<Item = Page, IntoIter = P>) -> Self {
- Self {
- page_iter: pages.into_iter(),
- }
- }
-}
-
-impl<P: Iterator<Item = Page>> PageReader for InMemoryPageReader<P> {
- fn get_next_page(&mut self) -> Result<Option<Page>> {
- Ok(self.page_iter.next())
- }
-}
-
-impl<P: Iterator<Item = Page>> Iterator for InMemoryPageReader<P> {
- type Item = Result<Page>;
-
- fn next(&mut self) -> Option<Self::Item> {
- self.get_next_page().transpose()
- }
-}
-
-/// A utility page iterator which stores page readers in memory, used for tests.
-#[derive(Clone)]
-pub struct InMemoryPageIterator<I: Iterator<Item = Vec<Page>>> {
- schema: SchemaDescPtr,
- column_desc: ColumnDescPtr,
- page_reader_iter: I,
-}
-
-impl<I: Iterator<Item = Vec<Page>>> InMemoryPageIterator<I> {
- pub fn new(
- schema: SchemaDescPtr,
- column_desc: ColumnDescPtr,
- pages: impl IntoIterator<Item = Vec<Page>, IntoIter = I>,
- ) -> Self {
- Self {
- schema,
- column_desc,
- page_reader_iter: pages.into_iter(),
- }
- }
-}
-
-impl<I: Iterator<Item = Vec<Page>>> Iterator for InMemoryPageIterator<I> {
- type Item = Result<Box<dyn PageReader>>;
-
- fn next(&mut self) -> Option<Self::Item> {
- self.page_reader_iter
- .next()
- .map(|x| Ok(Box::new(InMemoryPageReader::new(x)) as Box<dyn PageReader>))
- }
-}
-
-impl<I: Iterator<Item = Vec<Page>>> PageIterator for InMemoryPageIterator<I> {
- fn schema(&mut self) -> Result<SchemaDescPtr> {
- Ok(self.schema.clone())
- }
-
- fn column_schema(&mut self) -> Result<ColumnDescPtr> {
- Ok(self.column_desc.clone())
- }
-}
-
-pub fn make_pages<T: DataType>(
- desc: ColumnDescPtr,
- encoding: Encoding,
- num_pages: usize,
- levels_per_page: usize,
- min: T::T,
- max: T::T,
- def_levels: &mut Vec<i16>,
- rep_levels: &mut Vec<i16>,
- values: &mut Vec<T::T>,
- pages: &mut VecDeque<Page>,
- use_v2: bool,
-) where
- T::T: PartialOrd + SampleUniform + Copy,
-{
- let mut num_values = 0;
- let max_def_level = desc.max_def_level();
- let max_rep_level = desc.max_rep_level();
-
- let mem_tracker = Arc::new(MemTracker::new());
- let mut dict_encoder = DictEncoder::<T>::new(desc.clone(), mem_tracker);
-
- for i in 0..num_pages {
- let mut num_values_cur_page = 0;
- let level_range = i * levels_per_page..(i + 1) * levels_per_page;
-
- if max_def_level > 0 {
- random_numbers_range(levels_per_page, 0, max_def_level + 1, def_levels);
- for dl in &def_levels[level_range.clone()] {
- if *dl == max_def_level {
- num_values_cur_page += 1;
- }
- }
- } else {
- num_values_cur_page = levels_per_page;
- }
- if max_rep_level > 0 {
- random_numbers_range(levels_per_page, 0, max_rep_level + 1, rep_levels);
- }
- random_numbers_range(num_values_cur_page, min, max, values);
-
- // Generate the current page
-
- let mut pb =
- DataPageBuilderImpl::new(desc.clone(), num_values_cur_page as u32, use_v2);
- if max_rep_level > 0 {
- pb.add_rep_levels(max_rep_level, &rep_levels[level_range.clone()]);
- }
- if max_def_level > 0 {
- pb.add_def_levels(max_def_level, &def_levels[level_range]);
- }
-
- let value_range = num_values..num_values + num_values_cur_page;
- match encoding {
- Encoding::PLAIN_DICTIONARY | Encoding::RLE_DICTIONARY => {
- let _ = dict_encoder.put(&values[value_range.clone()]);
- let indices = dict_encoder
- .write_indices()
- .expect("write_indices() should be OK");
- pb.add_indices(indices);
- }
- Encoding::PLAIN => {
- pb.add_values::<T>(encoding, &values[value_range]);
- }
- enc => panic!("Unexpected encoding {}", enc),
- }
-
- let data_page = pb.consume();
- pages.push_back(data_page);
- num_values += num_values_cur_page;
- }
-
- if encoding == Encoding::PLAIN_DICTIONARY || encoding == Encoding::RLE_DICTIONARY {
- let dict = dict_encoder
- .write_dict()
- .expect("write_dict() should be OK");
- let dict_page = Page::DictionaryPage {
- buf: dict,
- num_values: dict_encoder.num_entries() as u32,
- encoding: Encoding::RLE_DICTIONARY,
- is_sorted: false,
- };
- pages.push_front(dict_page);
- }
-}
diff --git a/parquet/src/util/test_common/rand_gen.rs b/parquet/src/util/test_common/rand_gen.rs
deleted file mode 100644
index ea91b28..0000000
--- a/parquet/src/util/test_common/rand_gen.rs
+++ /dev/null
@@ -1,139 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-use rand::{
- distributions::{uniform::SampleUniform, Distribution, Standard},
- thread_rng, Rng,
-};
-
-use crate::data_type::*;
-use crate::util::memory::ByteBufferPtr;
-
-/// Random generator of data type `T` values and sequences.
-pub trait RandGen<T: DataType> {
- fn gen(len: i32) -> T::T;
-
- fn gen_vec(len: i32, total: usize) -> Vec<T::T> {
- let mut result = vec![];
- for _ in 0..total {
- result.push(Self::gen(len))
- }
- result
- }
-}
-
-impl RandGen<BoolType> for BoolType {
- fn gen(_: i32) -> bool {
- thread_rng().gen::<bool>()
- }
-}
-
-impl RandGen<Int32Type> for Int32Type {
- fn gen(_: i32) -> i32 {
- thread_rng().gen::<i32>()
- }
-}
-
-impl RandGen<Int64Type> for Int64Type {
- fn gen(_: i32) -> i64 {
- thread_rng().gen::<i64>()
- }
-}
-
-impl RandGen<Int96Type> for Int96Type {
- fn gen(_: i32) -> Int96 {
- let mut rng = thread_rng();
- let mut result = Int96::new();
- result.set_data(rng.gen::<u32>(), rng.gen::<u32>(), rng.gen::<u32>());
- result
- }
-}
-
-impl RandGen<FloatType> for FloatType {
- fn gen(_: i32) -> f32 {
- thread_rng().gen::<f32>()
- }
-}
-
-impl RandGen<DoubleType> for DoubleType {
- fn gen(_: i32) -> f64 {
- thread_rng().gen::<f64>()
- }
-}
-
-impl RandGen<ByteArrayType> for ByteArrayType {
- fn gen(_: i32) -> ByteArray {
- let mut rng = thread_rng();
- let mut result = ByteArray::new();
- let mut value = vec![];
- let len = rng.gen_range(0..128);
- for _ in 0..len {
- value.push(rng.gen_range(0..255));
- }
- result.set_data(ByteBufferPtr::new(value));
- result
- }
-}
-
-impl RandGen<FixedLenByteArrayType> for FixedLenByteArrayType {
- fn gen(len: i32) -> FixedLenByteArray {
- let mut rng = thread_rng();
- let value_len = if len < 0 {
- rng.gen_range(0..128)
- } else {
- len as usize
- };
- let value = random_bytes(value_len);
- ByteArray::from(value).into()
- }
-}
-
-pub fn random_bytes(n: usize) -> Vec<u8> {
- let mut result = vec![];
- let mut rng = thread_rng();
- for _ in 0..n {
- result.push(rng.gen_range(0..255));
- }
- result
-}
-
-pub fn random_bools(n: usize) -> Vec<bool> {
- let mut result = vec![];
- let mut rng = thread_rng();
- for _ in 0..n {
- result.push(rng.gen::<bool>());
- }
- result
-}
-
-pub fn random_numbers<T>(n: usize) -> Vec<T>
-where
- Standard: Distribution<T>,
-{
- let mut rng = thread_rng();
- Standard.sample_iter(&mut rng).take(n).collect()
-}
-
-pub fn random_numbers_range<T>(n: usize, low: T, high: T, result: &mut Vec<T>)
-where
- T: PartialOrd + SampleUniform + Copy,
-{
- let mut rng = thread_rng();
- for _ in 0..n {
- result.push(rng.gen_range(low..high));
- }
-}
diff --git a/parquet/tests/boolean_writer.rs b/parquet/tests/boolean_writer.rs
deleted file mode 100644
index b9d757e..0000000
--- a/parquet/tests/boolean_writer.rs
+++ /dev/null
@@ -1,100 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-use parquet::column::writer::ColumnWriter;
-use parquet::file::properties::WriterProperties;
-use parquet::file::reader::FileReader;
-use parquet::file::serialized_reader::SerializedFileReader;
-use parquet::file::writer::FileWriter;
-use parquet::file::writer::SerializedFileWriter;
-use parquet::schema::parser::parse_message_type;
-use std::fs;
-use std::path::Path;
-use std::sync::{mpsc, Arc};
-use std::thread;
-use std::time::Duration;
-
-#[test]
-fn it_writes_data_without_hanging() {
- let path = Path::new("it_writes_data_without_hanging.parquet");
-
- let message_type = "
- message BooleanType {
- REQUIRED BOOLEAN DIM0;
- }
-";
- let schema = Arc::new(parse_message_type(message_type).expect("parse schema"));
- let props = Arc::new(WriterProperties::builder().build());
- let file = fs::File::create(&path).expect("create file");
- let mut writer =
- SerializedFileWriter::new(file, schema, props).expect("create parquet writer");
- for _group in 0..1 {
- let mut row_group_writer = writer.next_row_group().expect("get row group writer");
- let values: Vec<i64> = vec![0; 2049];
- let my_bool_values: Vec<bool> = values
- .iter()
- .enumerate()
- .map(|(count, _x)| count % 2 == 0)
- .collect();
- while let Some(mut col_writer) =
- row_group_writer.next_column().expect("next column")
- {
- match col_writer {
- ColumnWriter::BoolColumnWriter(ref mut typed_writer) => {
- typed_writer
- .write_batch(&my_bool_values, None, None)
- .expect("writing bool column");
- }
- _ => {
- panic!("only test boolean values");
- }
- }
- row_group_writer
- .close_column(col_writer)
- .expect("close column");
- }
- let rg_md = row_group_writer.close().expect("close row group");
- println!("total rows written: {}", rg_md.num_rows());
- writer
- .close_row_group(row_group_writer)
- .expect("close row groups");
- }
- writer.close().expect("close writer");
-
- let bytes = fs::read(&path).expect("read file");
- assert_eq!(&bytes[0..4], &[b'P', b'A', b'R', b'1']);
-
- // Now that we have written our data and are happy with it, make
- // sure we can read it back in < 5 seconds...
- let (sender, receiver) = mpsc::channel();
- let _t = thread::spawn(move || {
- let file = fs::File::open(&Path::new("it_writes_data_without_hanging.parquet"))
- .expect("open file");
- let reader = SerializedFileReader::new(file).expect("get serialized reader");
- let iter = reader.get_row_iter(None).expect("get iterator");
- for record in iter {
- println!("reading: {}", record);
- }
- println!("finished reading");
- if let Ok(()) = sender.send(true) {}
- });
- assert_ne!(
- Err(mpsc::RecvTimeoutError::Timeout),
- receiver.recv_timeout(Duration::from_millis(5000))
- );
- fs::remove_file("it_writes_data_without_hanging.parquet").expect("remove file");
-}
diff --git a/parquet/tests/custom_writer.rs b/parquet/tests/custom_writer.rs
deleted file mode 100644
index 0a57e79..0000000
--- a/parquet/tests/custom_writer.rs
+++ /dev/null
@@ -1,100 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-use std::fs::File;
-use std::{
- fs,
- io::{prelude::*, SeekFrom},
- sync::Arc,
-};
-
-use parquet::file::writer::TryClone;
-use parquet::{
- basic::Repetition, basic::Type, file::properties::WriterProperties,
- file::writer::SerializedFileWriter, schema::types,
-};
-use std::env;
-
-// Test creating some sort of custom writer to ensure the
-// appropriate traits are exposed
-struct CustomWriter {
- file: File,
-}
-
-impl Write for CustomWriter {
- fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
- self.file.write(buf)
- }
- fn flush(&mut self) -> std::io::Result<()> {
- self.file.flush()
- }
-}
-
-impl Seek for CustomWriter {
- fn seek(&mut self, pos: SeekFrom) -> std::io::Result<u64> {
- self.file.seek(pos)
- }
-}
-
-impl TryClone for CustomWriter {
- fn try_clone(&self) -> std::io::Result<Self> {
- use std::io::{Error, ErrorKind};
- Err(Error::new(ErrorKind::Other, "Clone not supported"))
- }
-}
-
-#[test]
-fn test_custom_writer() {
- let schema = Arc::new(
- types::Type::group_type_builder("schema")
- .with_fields(&mut vec![Arc::new(
- types::Type::primitive_type_builder("col1", Type::INT32)
- .with_repetition(Repetition::REQUIRED)
- .build()
- .unwrap(),
- )])
- .build()
- .unwrap(),
- );
- let props = Arc::new(WriterProperties::builder().build());
-
- let file = get_temp_file("test_custom_file_writer");
- let test_file = file.try_clone().unwrap();
-
- let writer = CustomWriter { file };
-
- // test is that this file can be created
- let file_writer = SerializedFileWriter::new(writer, schema, props).unwrap();
- std::mem::drop(file_writer);
-
- // ensure the file now exists and has non zero size
- let metadata = test_file.metadata().unwrap();
- assert!(metadata.len() > 0);
-}
-
-/// Returns file handle for a temp file in 'target' directory with a provided content
-fn get_temp_file(file_name: &str) -> fs::File {
- // build tmp path to a file in "target/debug/testdata"
- let mut path_buf = env::current_dir().unwrap();
- path_buf.push("target");
- path_buf.push("debug");
- path_buf.push("testdata");
- fs::create_dir_all(&path_buf).unwrap();
- path_buf.push(file_name);
-
- File::create(path_buf).unwrap()
-}
diff --git a/parquet_derive/Cargo.toml b/parquet_derive/Cargo.toml
deleted file mode 100644
index 1d16137..0000000
--- a/parquet_derive/Cargo.toml
+++ /dev/null
@@ -1,42 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-[package]
-name = "parquet_derive"
-version = "5.0.0-SNAPSHOT"
-license = "Apache-2.0"
-description = "Derive macros for the Rust implementation of Apache Parquet"
-homepage = "https://github.com/apache/arrow-rs"
-repository = "https://github.com/apache/arrow-rs"
-authors = ["Apache Arrow <dev@arrow.apache.org>"]
-keywords = [ "parquet" ]
-readme = "README.md"
-edition = "2018"
-
-[lib]
-proc-macro = true
-
-[features]
-chrono = []
-bigdecimal = []
-uuid = []
-
-[dependencies]
-proc-macro2 = "1.0"
-quote = "1.0"
-syn = { version = "1.0", features = ["full", "extra-traits"] }
-parquet = { path = "../parquet", version = "5.0.0-SNAPSHOT" }
diff --git a/parquet_derive/README.md b/parquet_derive/README.md
deleted file mode 100644
index 47ac8ae..0000000
--- a/parquet_derive/README.md
+++ /dev/null
@@ -1,98 +0,0 @@
-<!---
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
--->
-
-# Parquet Derive
-
-A crate for deriving `RecordWriter` for arbitrary, _simple_ structs. This does not generate writers for arbitrarily nested
-structures. It only works for primitives and a few generic structures and
-various levels of reference. Please see features checklist for what is currently
-supported.
-
-Derive also has some support for the chrono time library. You must must enable the `chrono` feature to get this support.
-
-## Usage
-Add this to your Cargo.toml:
-```toml
-[dependencies]
-parquet = "4.0.0-SNAPSHOT"
-parquet_derive = "4.0.0-SNAPSHOT"
-```
-
-and this to your crate root:
-```rust
-extern crate parquet;
-#[macro_use] extern crate parquet_derive;
-```
-
-Example usage of deriving a `RecordWriter` for your struct:
-
-```rust
-use parquet;
-use parquet::record::RecordWriter;
-
-#[derive(ParquetRecordWriter)]
-struct ACompleteRecord<'a> {
- pub a_bool: bool,
- pub a_str: &'a str,
- pub a_string: String,
- pub a_borrowed_string: &'a String,
- pub maybe_a_str: Option<&'a str>,
- pub magic_number: i32,
- pub low_quality_pi: f32,
- pub high_quality_pi: f64,
- pub maybe_pi: Option<f32>,
- pub maybe_best_pi: Option<f64>,
-}
-
-// Initialize your parquet file
-let mut writer = SerializedFileWriter::new(file, schema, props).unwrap();
-let mut row_group = writer.next_row_group().unwrap();
-
-// Build up your records
-let chunks = vec![ACompleteRecord{...}];
-
-// The derived `RecordWriter` takes over here
-(&chunks[..]).write_to_row_group(&mut row_group);
-
-writer.close_row_group(row_group).unwrap();
-writer.close().unwrap();
-```
-
-## Features
-- [X] Support writing `String`, `&str`, `bool`, `i32`, `f32`, `f64`, `Vec<u8>`
-- [ ] Support writing dictionaries
-- [X] Support writing logical types like timestamp
-- [X] Derive definition_levels for `Option`
-- [ ] Derive definition levels for nested structures
-- [ ] Derive writing tuple struct
-- [ ] Derive writing `tuple` container types
-
-## Requirements
-- Same as `parquet-rs`
-
-## Test
-Testing a `*_derive` crate requires an intermediate crate. Go to `parquet_derive_test` and run `cargo test` for
-unit tests.
-
-## Docs
-To build documentation, run `cargo doc --no-deps`.
-To compile and view in the browser, run `cargo doc --no-deps --open`.
-
-## License
-Licensed under the Apache License, Version 2.0: http://www.apache.org/licenses/LICENSE-2.0.
diff --git a/parquet_derive/src/lib.rs b/parquet_derive/src/lib.rs
deleted file mode 100644
index 279d0f7..0000000
--- a/parquet_derive/src/lib.rs
+++ /dev/null
@@ -1,126 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#![recursion_limit = "128"]
-
-extern crate proc_macro;
-extern crate proc_macro2;
-extern crate syn;
-#[macro_use]
-extern crate quote;
-
-extern crate parquet;
-
-use syn::{parse_macro_input, Data, DataStruct, DeriveInput};
-
-mod parquet_field;
-
-/// Derive flat, simple RecordWriter implementations. Works by parsing
-/// a struct tagged with `#[derive(ParquetRecordWriter)]` and emitting
-/// the correct writing code for each field of the struct. Column writers
-/// are generated in the order they are defined.
-///
-/// It is up to the programmer to keep the order of the struct
-/// fields lined up with the schema.
-///
-/// Example:
-///
-/// ```ignore
-/// use parquet;
-/// use parquet::record::RecordWriter;
-/// use parquet::schema::parser::parse_message_type;
-///
-/// use std::sync::Arc;
-//
-/// #[derive(ParquetRecordWriter)]
-/// struct ACompleteRecord<'a> {
-/// pub a_bool: bool,
-/// pub a_str: &'a str,
-/// }
-///
-/// let schema_str = "message schema {
-/// REQUIRED boolean a_bool;
-/// REQUIRED BINARY a_str (UTF8);
-/// }";
-///
-/// pub fn write_some_records() {
-/// let samples = vec![
-/// ACompleteRecord {
-/// a_bool: true,
-/// a_str: "I'm true"
-/// },
-/// ACompleteRecord {
-/// a_bool: false,
-/// a_str: "I'm false"
-/// }
-/// ];
-///
-/// let schema = Arc::new(parse_message_type(schema_str).unwrap());
-///
-/// let props = Arc::new(WriterProperties::builder().build());
-/// let mut writer = SerializedFileWriter::new(file, schema, props).unwrap();
-///
-/// let mut row_group = writer.next_row_group().unwrap();
-/// samples.as_slice().write_to_row_group(&mut row_group).unwrap();
-/// writer.close_row_group(row_group).unwrap();
-/// writer.close().unwrap();
-/// }
-/// ```
-///
-#[proc_macro_derive(ParquetRecordWriter)]
-pub fn parquet_record_writer(input: proc_macro::TokenStream) -> proc_macro::TokenStream {
- let input: DeriveInput = parse_macro_input!(input as DeriveInput);
- let fields = match input.data {
- Data::Struct(DataStruct { fields, .. }) => fields,
- Data::Enum(_) => unimplemented!("Enum currently is not supported"),
- Data::Union(_) => unimplemented!("Union currently is not supported"),
- };
-
- let field_infos: Vec<_> = fields
- .iter()
- .map(|f: &syn::Field| parquet_field::Field::from(f))
- .collect();
-
- let writer_snippets: Vec<proc_macro2::TokenStream> =
- field_infos.iter().map(|x| x.writer_snippet()).collect();
-
- let derived_for = input.ident;
- let generics = input.generics;
-
- (quote! {
- impl#generics RecordWriter<#derived_for#generics> for &[#derived_for#generics] {
- fn write_to_row_group(&self, row_group_writer: &mut Box<parquet::file::writer::RowGroupWriter>) -> Result<(), parquet::errors::ParquetError> {
- let mut row_group_writer = row_group_writer;
- let records = &self; // Used by all the writer snippets to be more clear
-
- #(
- {
- let mut some_column_writer = row_group_writer.next_column().unwrap();
- if let Some(mut column_writer) = some_column_writer {
- #writer_snippets
- row_group_writer.close_column(column_writer)?;
- } else {
- return Err(parquet::errors::ParquetError::General("Failed to get next column".into()))
- }
- }
- );*
-
- Ok(())
- }
- }
- }).into()
-}
diff --git a/parquet_derive/src/parquet_field.rs b/parquet_derive/src/parquet_field.rs
deleted file mode 100644
index 328f4a6..0000000
--- a/parquet_derive/src/parquet_field.rs
+++ /dev/null
@@ -1,920 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#[derive(Debug, PartialEq)]
-pub struct Field {
- ident: syn::Ident,
- ty: Type,
- is_a_byte_buf: bool,
- third_party_type: Option<ThirdPartyType>,
-}
-
-/// Use third party libraries, detected
-/// at compile time. These libraries will
-/// be written to parquet as their preferred
-/// physical type.
-///
-/// ChronoNaiveDateTime is written as i64
-/// ChronoNaiveDate is written as i32
-#[derive(Debug, PartialEq)]
-enum ThirdPartyType {
- ChronoNaiveDateTime,
- ChronoNaiveDate,
- Uuid,
-}
-
-impl Field {
- pub fn from(f: &syn::Field) -> Self {
- let ty = Type::from(f);
- let is_a_byte_buf = ty.physical_type() == parquet::basic::Type::BYTE_ARRAY;
-
- let third_party_type = match &ty.last_part()[..] {
- "NaiveDateTime" => Some(ThirdPartyType::ChronoNaiveDateTime),
- "NaiveDate" => Some(ThirdPartyType::ChronoNaiveDate),
- "Uuid" => Some(ThirdPartyType::Uuid),
- _ => None,
- };
-
- Field {
- ident: f
- .ident
- .clone()
- .expect("Only structs with named fields are currently supported"),
- ty,
- is_a_byte_buf,
- third_party_type,
- }
- }
-
- /// Takes the parsed field of the struct and emits a valid
- /// column writer snippet. Should match exactly what you
- /// would write by hand.
- ///
- /// Can only generate writers for basic structs, for example:
- ///
- /// struct Record {
- /// a_bool: bool,
- /// maybe_a_bool: Option<bool>
- /// }
- ///
- /// but not
- ///
- /// struct UnsupportedNestedRecord {
- /// a_property: bool,
- /// nested_record: Record
- /// }
- ///
- /// because this parsing logic is not sophisticated enough for definition
- /// levels beyond 2.
- pub fn writer_snippet(&self) -> proc_macro2::TokenStream {
- let ident = &self.ident;
- let column_writer = self.ty.column_writer();
-
- let vals_builder = match &self.ty {
- Type::TypePath(_) => self.copied_direct_vals(),
- Type::Option(ref first_type) => match **first_type {
- Type::TypePath(_) => self.option_into_vals(),
- Type::Reference(_, ref second_type) => match **second_type {
- Type::TypePath(_) => self.option_into_vals(),
- _ => unimplemented!("Unsupported type encountered"),
- },
- ref f => unimplemented!("Unsupported: {:#?}", f),
- },
- Type::Reference(_, ref first_type) => match **first_type {
- Type::TypePath(_) => self.copied_direct_vals(),
- Type::Option(ref second_type) => match **second_type {
- Type::TypePath(_) => self.option_into_vals(),
- Type::Reference(_, ref second_type) => match **second_type {
- Type::TypePath(_) => self.option_into_vals(),
- _ => unimplemented!("Unsupported type encountered"),
- },
- ref f => unimplemented!("Unsupported: {:#?}", f),
- },
- ref f => unimplemented!("Unsupported: {:#?}", f),
- },
- f => unimplemented!("Unsupported: {:#?}", f),
- };
-
- let definition_levels = match &self.ty {
- Type::TypePath(_) => None,
- Type::Option(ref first_type) => match **first_type {
- Type::TypePath(_) => Some(self.optional_definition_levels()),
- Type::Option(_) => unimplemented!("Unsupported nesting encountered"),
- Type::Reference(_, ref second_type)
- | Type::Vec(ref second_type)
- | Type::Array(ref second_type) => match **second_type {
- Type::TypePath(_) => Some(self.optional_definition_levels()),
- _ => unimplemented!("Unsupported nesting encountered"),
- },
- },
- Type::Reference(_, ref first_type)
- | Type::Vec(ref first_type)
- | Type::Array(ref first_type) => match **first_type {
- Type::TypePath(_) => None,
- Type::Reference(_, ref second_type)
- | Type::Vec(ref second_type)
- | Type::Array(ref second_type)
- | Type::Option(ref second_type) => match **second_type {
- Type::TypePath(_) => Some(self.optional_definition_levels()),
- Type::Reference(_, ref third_type) => match **third_type {
- Type::TypePath(_) => Some(self.optional_definition_levels()),
- _ => unimplemented!("Unsupported definition encountered"),
- },
- _ => unimplemented!("Unsupported definition encountered"),
- },
- },
- };
-
- // "vals" is the run of primitive data being written for the column
- // "definition_levels" is a vector of bools which controls whether a value is missing or present
- // this TokenStream is only one part of the code for writing a column and
- // it relies on values calculated in prior code snippets, namely "definition_levels" and "vals_builder".
- // All the context is put together in this functions final quote and
- // this expression just switches between non-nullable and nullable write statements
- let write_batch_expr = if definition_levels.is_some() {
- quote! {
- if let #column_writer(ref mut typed) = column_writer {
- typed.write_batch(&vals[..], Some(&definition_levels[..]), None)?;
- } else {
- panic!("Schema and struct disagree on type for {}", stringify!{#ident})
- }
- }
- } else {
- quote! {
- if let #column_writer(ref mut typed) = column_writer {
- typed.write_batch(&vals[..], None, None)?;
- } else {
- panic!("Schema and struct disagree on type for {}", stringify!{#ident})
- }
- }
- };
-
- quote! {
- {
- #definition_levels
-
- #vals_builder
-
- #write_batch_expr
- }
- }
- }
-
- fn option_into_vals(&self) -> proc_macro2::TokenStream {
- let field_name = &self.ident;
- let is_a_byte_buf = self.is_a_byte_buf;
- let is_a_timestamp =
- self.third_party_type == Some(ThirdPartyType::ChronoNaiveDateTime);
- let is_a_date = self.third_party_type == Some(ThirdPartyType::ChronoNaiveDate);
- let is_a_uuid = self.third_party_type == Some(ThirdPartyType::Uuid);
- let copy_to_vec = !matches!(
- self.ty.physical_type(),
- parquet::basic::Type::BYTE_ARRAY | parquet::basic::Type::FIXED_LEN_BYTE_ARRAY
- );
-
- let binding = if copy_to_vec {
- quote! { let Some(inner) = rec.#field_name }
- } else {
- quote! { let Some(ref inner) = rec.#field_name }
- };
-
- let some = if is_a_timestamp {
- quote! { Some(inner.timestamp_millis()) }
- } else if is_a_date {
- quote! { Some(inner.signed_duration_since(chrono::NaiveDate::from_ymd(1970, 1, 1)).num_days() as i32) }
- } else if is_a_uuid {
- quote! { Some((&inner.to_string()[..]).into()) }
- } else if is_a_byte_buf {
- quote! { Some((&inner[..]).into())}
- } else {
- quote! { Some(inner) }
- };
-
- quote! {
- let vals: Vec<_> = records.iter().filter_map(|rec| {
- if #binding {
- #some
- } else {
- None
- }
- }).collect();
- }
- }
-
- fn copied_direct_vals(&self) -> proc_macro2::TokenStream {
- let field_name = &self.ident;
- let is_a_byte_buf = self.is_a_byte_buf;
- let is_a_timestamp =
- self.third_party_type == Some(ThirdPartyType::ChronoNaiveDateTime);
- let is_a_date = self.third_party_type == Some(ThirdPartyType::ChronoNaiveDate);
- let is_a_uuid = self.third_party_type == Some(ThirdPartyType::Uuid);
-
- let access = if is_a_timestamp {
- quote! { rec.#field_name.timestamp_millis() }
- } else if is_a_date {
- quote! { rec.#field_name.signed_duration_since(chrono::NaiveDate::from_ymd(1970, 1, 1)).num_days() as i32 }
- } else if is_a_uuid {
- quote! { (&rec.#field_name.to_string()[..]).into() }
- } else if is_a_byte_buf {
- quote! { (&rec.#field_name[..]).into() }
- } else {
- quote! { rec.#field_name }
- };
-
- quote! {
- let vals: Vec<_> = records.iter().map(|rec| #access).collect();
- }
- }
-
- fn optional_definition_levels(&self) -> proc_macro2::TokenStream {
- let field_name = &self.ident;
-
- quote! {
- let definition_levels: Vec<i16> = self
- .iter()
- .map(|rec| if rec.#field_name.is_some() { 1 } else { 0 })
- .collect();
- }
- }
-}
-
-#[allow(clippy::enum_variant_names)]
-#[allow(clippy::large_enum_variant)]
-#[derive(Debug, PartialEq)]
-enum Type {
- Array(Box<Type>),
- Option(Box<Type>),
- Vec(Box<Type>),
- TypePath(syn::Type),
- Reference(Option<syn::Lifetime>, Box<Type>),
-}
-
-impl Type {
- /// Takes a rust type and returns the appropriate
- /// parquet-rs column writer
- fn column_writer(&self) -> syn::TypePath {
- use parquet::basic::Type as BasicType;
-
- match self.physical_type() {
- BasicType::BOOLEAN => {
- syn::parse_quote!(parquet::column::writer::ColumnWriter::BoolColumnWriter)
- }
- BasicType::INT32 => syn::parse_quote!(
- parquet::column::writer::ColumnWriter::Int32ColumnWriter
- ),
- BasicType::INT64 => syn::parse_quote!(
- parquet::column::writer::ColumnWriter::Int64ColumnWriter
- ),
- BasicType::INT96 => syn::parse_quote!(
- parquet::column::writer::ColumnWriter::Int96ColumnWriter
- ),
- BasicType::FLOAT => syn::parse_quote!(
- parquet::column::writer::ColumnWriter::FloatColumnWriter
- ),
- BasicType::DOUBLE => syn::parse_quote!(
- parquet::column::writer::ColumnWriter::DoubleColumnWriter
- ),
- BasicType::BYTE_ARRAY => syn::parse_quote!(
- parquet::column::writer::ColumnWriter::ByteArrayColumnWriter
- ),
- BasicType::FIXED_LEN_BYTE_ARRAY => syn::parse_quote!(
- parquet::column::writer::ColumnWriter::FixedLenByteArrayColumnWriter
- ),
- }
- }
-
- /// Helper to simplify a nested field definition to its leaf type
- ///
- /// Ex:
- /// Option<&String> => Type::TypePath(String)
- /// &Option<i32> => Type::TypePath(i32)
- /// Vec<Vec<u8>> => Type::Vec(u8)
- ///
- /// Useful in determining the physical type of a field and the
- /// definition levels.
- fn leaf_type_recursive(&self) -> &Type {
- self.leaf_type_recursive_helper(self, None)
- }
-
- fn leaf_type_recursive_helper<'a>(
- &'a self,
- ty: &'a Type,
- parent_ty: Option<&'a Type>,
- ) -> &Type {
- match ty {
- Type::TypePath(_) => parent_ty.unwrap_or(ty),
- Type::Option(ref first_type)
- | Type::Vec(ref first_type)
- | Type::Array(ref first_type)
- | Type::Reference(_, ref first_type) => {
- self.leaf_type_recursive_helper(first_type, Some(ty))
- }
- }
- }
-
- /// Helper method to further unwrap leaf_type() to get inner-most
- /// type information, useful for determining the physical type
- /// and normalizing the type paths.
- fn inner_type(&self) -> &syn::Type {
- let leaf_type = self.leaf_type_recursive();
-
- match leaf_type {
- Type::TypePath(ref type_) => type_,
- Type::Option(ref first_type)
- | Type::Vec(ref first_type)
- | Type::Array(ref first_type)
- | Type::Reference(_, ref first_type) => match **first_type {
- Type::TypePath(ref type_) => type_,
- _ => unimplemented!("leaf_type() should only return shallow types"),
- },
- }
- }
-
- /// Helper to normalize a type path by extracting the
- /// most identifiable part
- ///
- /// Ex:
- /// std::string::String => String
- /// Vec<u8> => Vec<u8>
- /// chrono::NaiveDateTime => NaiveDateTime
- ///
- /// Does run the risk of mis-identifying a type if import
- /// rename is in play. Please note procedural macros always
- /// run before type resolution so this is a risk the user
- /// takes on when renaming imports.
- fn last_part(&self) -> String {
- let inner_type = self.inner_type();
- let inner_type_str = (quote! { #inner_type }).to_string();
-
- inner_type_str
- .split("::")
- .last()
- .unwrap()
- .trim()
- .to_string()
- }
-
- /// Converts rust types to parquet physical types.
- ///
- /// Ex:
- /// [u8; 10] => FIXED_LEN_BYTE_ARRAY
- /// Vec<u8> => BYTE_ARRAY
- /// String => BYTE_ARRAY
- /// i32 => INT32
- fn physical_type(&self) -> parquet::basic::Type {
- use parquet::basic::Type as BasicType;
-
- let last_part = self.last_part();
- let leaf_type = self.leaf_type_recursive();
-
- match leaf_type {
- Type::Array(ref first_type) => {
- if let Type::TypePath(_) = **first_type {
- if last_part == "u8" {
- return BasicType::FIXED_LEN_BYTE_ARRAY;
- }
- }
- }
- Type::Vec(ref first_type) => {
- if let Type::TypePath(_) = **first_type {
- if last_part == "u8" {
- return BasicType::BYTE_ARRAY;
- }
- }
- }
- _ => (),
- }
-
- match last_part.trim() {
- "bool" => BasicType::BOOLEAN,
- "u8" | "u16" | "u32" => BasicType::INT32,
- "i8" | "i16" | "i32" | "NaiveDate" => BasicType::INT32,
- "u64" | "i64" | "usize" | "NaiveDateTime" => BasicType::INT64,
- "f32" => BasicType::FLOAT,
- "f64" => BasicType::DOUBLE,
- "String" | "str" | "Uuid" => BasicType::BYTE_ARRAY,
- f => unimplemented!("{} currently is not supported", f),
- }
- }
-
- /// Convert a parsed rust field AST in to a more easy to manipulate
- /// parquet_derive::Field
- fn from(f: &syn::Field) -> Self {
- Type::from_type(f, &f.ty)
- }
-
- fn from_type(f: &syn::Field, ty: &syn::Type) -> Self {
- match ty {
- syn::Type::Path(ref p) => Type::from_type_path(f, p),
- syn::Type::Reference(ref tr) => Type::from_type_reference(f, tr),
- syn::Type::Array(ref ta) => Type::from_type_array(f, ta),
- other => unimplemented!(
- "Unable to derive {:?} - it is currently an unsupported type\n{:#?}",
- f.ident.as_ref().unwrap(),
- other
- ),
- }
- }
-
- fn from_type_path(f: &syn::Field, p: &syn::TypePath) -> Self {
- let last_segment = p.path.segments.last().unwrap();
-
- let is_vec =
- last_segment.ident == syn::Ident::new("Vec", proc_macro2::Span::call_site());
- let is_option = last_segment.ident
- == syn::Ident::new("Option", proc_macro2::Span::call_site());
-
- if is_vec || is_option {
- let generic_type = match &last_segment.arguments {
- syn::PathArguments::AngleBracketed(angle_args) => {
- assert_eq!(angle_args.args.len(), 1);
- let first_arg = &angle_args.args[0];
-
- match first_arg {
- syn::GenericArgument::Type(ref typath) => typath.clone(),
- other => unimplemented!("Unsupported: {:#?}", other),
- }
- }
- other => unimplemented!("Unsupported: {:#?}", other),
- };
-
- if is_vec {
- Type::Vec(Box::new(Type::from_type(f, &generic_type)))
- } else {
- Type::Option(Box::new(Type::from_type(f, &generic_type)))
- }
- } else {
- Type::TypePath(syn::Type::Path(p.clone()))
- }
- }
-
- fn from_type_reference(f: &syn::Field, tr: &syn::TypeReference) -> Self {
- let lifetime = tr.lifetime.clone();
- let inner_type = Type::from_type(f, tr.elem.as_ref());
- Type::Reference(lifetime, Box::new(inner_type))
- }
-
- fn from_type_array(f: &syn::Field, ta: &syn::TypeArray) -> Self {
- let inner_type = Type::from_type(f, ta.elem.as_ref());
- Type::Array(Box::new(inner_type))
- }
-}
-
-#[cfg(test)]
-mod test {
- use super::*;
- use syn::{self, Data, DataStruct, DeriveInput};
-
- fn extract_fields(input: proc_macro2::TokenStream) -> Vec<syn::Field> {
- let input: DeriveInput = syn::parse2(input).unwrap();
-
- let fields = match input.data {
- Data::Struct(DataStruct { fields, .. }) => fields,
- _ => panic!("Input must be a struct"),
- };
-
- fields.iter().map(|field| field.to_owned()).collect()
- }
-
- #[test]
- fn test_generating_a_simple_writer_snippet() {
- let snippet: proc_macro2::TokenStream = quote! {
- struct ABoringStruct {
- counter: usize,
- }
- };
-
- let fields = extract_fields(snippet);
- let counter = Field::from(&fields[0]);
-
- let snippet = counter.writer_snippet().to_string();
- assert_eq!(snippet,
- (quote!{
- {
- let vals : Vec < _ > = records . iter ( ) . map ( | rec | rec . counter ) . collect ( );
-
- if let parquet::column::writer::ColumnWriter::Int64ColumnWriter ( ref mut typed ) = column_writer {
- typed . write_batch ( & vals [ .. ] , None , None ) ?;
- } else {
- panic!("Schema and struct disagree on type for {}" , stringify!{ counter } )
- }
- }
- }).to_string()
- )
- }
-
- #[test]
- fn test_optional_to_writer_snippet() {
- let struct_def: proc_macro2::TokenStream = quote! {
- struct StringBorrower<'a> {
- optional_str: Option<&'a str>,
- optional_string: &Option<String>,
- optional_dumb_int: &Option<&i32>,
- }
- };
-
- let fields = extract_fields(struct_def);
-
- let optional = Field::from(&fields[0]);
- let snippet = optional.writer_snippet();
- assert_eq!(snippet.to_string(),
- (quote! {
- {
- let definition_levels : Vec < i16 > = self . iter ( ) . map ( | rec | if rec . optional_str . is_some ( ) { 1 } else { 0 } ) . collect ( ) ;
-
- let vals: Vec <_> = records.iter().filter_map( |rec| {
- if let Some ( ref inner ) = rec . optional_str {
- Some ( (&inner[..]).into() )
- } else {
- None
- }
- }).collect();
-
- if let parquet::column::writer::ColumnWriter::ByteArrayColumnWriter ( ref mut typed ) = column_writer {
- typed . write_batch ( & vals [ .. ] , Some(&definition_levels[..]) , None ) ? ;
- } else {
- panic!("Schema and struct disagree on type for {}" , stringify ! { optional_str } )
- }
- }
- }
- ).to_string());
-
- let optional = Field::from(&fields[1]);
- let snippet = optional.writer_snippet();
- assert_eq!(snippet.to_string(),
- (quote!{
- {
- let definition_levels : Vec < i16 > = self . iter ( ) . map ( | rec | if rec . optional_string . is_some ( ) { 1 } else { 0 } ) . collect ( ) ;
-
- let vals: Vec <_> = records.iter().filter_map( |rec| {
- if let Some ( ref inner ) = rec . optional_string {
- Some ( (&inner[..]).into() )
- } else {
- None
- }
- }).collect();
-
- if let parquet::column::writer::ColumnWriter::ByteArrayColumnWriter ( ref mut typed ) = column_writer {
- typed . write_batch ( & vals [ .. ] , Some(&definition_levels[..]) , None ) ? ;
- } else {
- panic!("Schema and struct disagree on type for {}" , stringify ! { optional_string } )
- }
- }
- }).to_string());
-
- let optional = Field::from(&fields[2]);
- let snippet = optional.writer_snippet();
- assert_eq!(snippet.to_string(),
- (quote!{
- {
- let definition_levels : Vec < i16 > = self . iter ( ) . map ( | rec | if rec . optional_dumb_int . is_some ( ) { 1 } else { 0 } ) . collect ( ) ;
-
- let vals: Vec <_> = records.iter().filter_map( |rec| {
- if let Some ( inner ) = rec . optional_dumb_int {
- Some ( inner )
- } else {
- None
- }
- }).collect();
-
- if let parquet::column::writer::ColumnWriter::Int32ColumnWriter ( ref mut typed ) = column_writer {
- typed . write_batch ( & vals [ .. ] , Some(&definition_levels[..]) , None ) ? ;
- } else {
- panic!("Schema and struct disagree on type for {}" , stringify ! { optional_dumb_int } )
- }
- }
- }).to_string());
- }
-
- #[test]
- fn test_converting_to_column_writer_type() {
- let snippet: proc_macro2::TokenStream = quote! {
- struct ABasicStruct {
- yes_no: bool,
- name: String,
- }
- };
-
- let fields = extract_fields(snippet);
- let processed: Vec<_> = fields.iter().map(|field| Field::from(field)).collect();
-
- let column_writers: Vec<_> = processed
- .iter()
- .map(|field| field.ty.column_writer())
- .collect();
-
- assert_eq!(
- column_writers,
- vec![
- syn::parse_quote!(
- parquet::column::writer::ColumnWriter::BoolColumnWriter
- ),
- syn::parse_quote!(
- parquet::column::writer::ColumnWriter::ByteArrayColumnWriter
- )
- ]
- );
- }
-
- #[test]
- fn convert_basic_struct() {
- let snippet: proc_macro2::TokenStream = quote! {
- struct ABasicStruct {
- yes_no: bool,
- name: String,
- }
- };
-
- let fields = extract_fields(snippet);
- let processed: Vec<_> = fields.iter().map(|field| Field::from(field)).collect();
- assert_eq!(processed.len(), 2);
-
- assert_eq!(
- processed,
- vec![
- Field {
- ident: syn::Ident::new("yes_no", proc_macro2::Span::call_site()),
- ty: Type::TypePath(syn::parse_quote!(bool)),
- is_a_byte_buf: false,
- third_party_type: None,
- },
- Field {
- ident: syn::Ident::new("name", proc_macro2::Span::call_site()),
- ty: Type::TypePath(syn::parse_quote!(String)),
- is_a_byte_buf: true,
- third_party_type: None,
- }
- ]
- )
- }
-
- #[test]
- fn test_get_inner_type() {
- let snippet: proc_macro2::TokenStream = quote! {
- struct LotsOfInnerTypes {
- a_vec: Vec<u8>,
- a_option: std::option::Option<bool>,
- a_silly_string: std::string::String,
- a_complicated_thing: std::option::Option<std::result::Result<(),()>>,
- }
- };
-
- let fields = extract_fields(snippet);
- let converted_fields: Vec<_> =
- fields.iter().map(|field| Type::from(field)).collect();
- let inner_types: Vec<_> = converted_fields
- .iter()
- .map(|field| field.inner_type())
- .collect();
- let inner_types_strs: Vec<_> = inner_types
- .iter()
- .map(|ty| (quote! { #ty }).to_string())
- .collect();
-
- assert_eq!(
- inner_types_strs,
- vec![
- "u8",
- "bool",
- "std :: string :: String",
- "std :: result :: Result < () , () >"
- ]
- )
- }
-
- #[test]
- fn test_physical_type() {
- use parquet::basic::Type as BasicType;
- let snippet: proc_macro2::TokenStream = quote! {
- struct LotsOfInnerTypes {
- a_buf: Vec<u8>,
- a_number: i32,
- a_verbose_option: std::option::Option<bool>,
- a_silly_string: std::string::String,
- a_fix_byte_buf: [u8; 10],
- a_complex_option: Option<&Vec<u8>>,
- a_complex_vec: &Vec<&Option<u8>>,
- }
- };
-
- let fields = extract_fields(snippet);
- let converted_fields: Vec<_> =
- fields.iter().map(|field| Type::from(field)).collect();
- let physical_types: Vec<_> = converted_fields
- .iter()
- .map(|ty| ty.physical_type())
- .collect();
-
- assert_eq!(
- physical_types,
- vec![
- BasicType::BYTE_ARRAY,
- BasicType::INT32,
- BasicType::BOOLEAN,
- BasicType::BYTE_ARRAY,
- BasicType::FIXED_LEN_BYTE_ARRAY,
- BasicType::BYTE_ARRAY,
- BasicType::INT32
- ]
- )
- }
-
- #[test]
- fn test_convert_comprehensive_owned_struct() {
- let snippet: proc_macro2::TokenStream = quote! {
- struct VecHolder {
- a_vec: Vec<u8>,
- a_option: std::option::Option<bool>,
- a_silly_string: std::string::String,
- a_complicated_thing: std::option::Option<std::result::Result<(),()>>,
- }
- };
-
- let fields = extract_fields(snippet);
- let converted_fields: Vec<_> =
- fields.iter().map(|field| Type::from(field)).collect();
-
- assert_eq!(
- converted_fields,
- vec![
- Type::Vec(Box::new(Type::TypePath(syn::parse_quote!(u8)))),
- Type::Option(Box::new(Type::TypePath(syn::parse_quote!(bool)))),
- Type::TypePath(syn::parse_quote!(std::string::String)),
- Type::Option(Box::new(Type::TypePath(
- syn::parse_quote!(std::result::Result<(),()>)
- ))),
- ]
- );
- }
-
- #[test]
- fn test_convert_borrowed_struct() {
- let snippet: proc_macro2::TokenStream = quote! {
- struct Borrower<'a> {
- a_str: &'a str,
- a_borrowed_option: &'a Option<bool>,
- so_many_borrows: &'a Option<&'a str>,
- }
- };
-
- let fields = extract_fields(snippet);
- let types: Vec<_> = fields.iter().map(|field| Type::from(field)).collect();
-
- assert_eq!(
- types,
- vec![
- Type::Reference(
- Some(syn::Lifetime::new("'a", proc_macro2::Span::call_site())),
- Box::new(Type::TypePath(syn::parse_quote!(str)))
- ),
- Type::Reference(
- Some(syn::Lifetime::new("'a", proc_macro2::Span::call_site())),
- Box::new(Type::Option(Box::new(Type::TypePath(syn::parse_quote!(
- bool
- )))))
- ),
- Type::Reference(
- Some(syn::Lifetime::new("'a", proc_macro2::Span::call_site())),
- Box::new(Type::Option(Box::new(Type::Reference(
- Some(syn::Lifetime::new("'a", proc_macro2::Span::call_site())),
- Box::new(Type::TypePath(syn::parse_quote!(str)))
- ))))
- ),
- ]
- );
- }
-
- #[test]
- #[cfg(feature = "chrono")]
- fn test_chrono_timestamp_millis() {
- let snippet: proc_macro2::TokenStream = quote! {
- struct ATimestampStruct {
- henceforth: chrono::NaiveDateTime,
- maybe_happened: Option<&chrono::NaiveDateTime>,
- }
- };
-
- let fields = extract_fields(snippet);
- let when = Field::from(&fields[0]);
- assert_eq!(when.writer_snippet().to_string(),(quote!{
- {
- let vals : Vec<_> = records.iter().map(|rec| rec.henceforth.timestamp_millis() ).collect();
- if let parquet::column::writer::ColumnWriter::Int64ColumnWriter(ref mut typed) = column_writer {
- typed.write_batch(&vals[..], None, None) ?;
- } else {
- panic!("Schema and struct disagree on type for {}" , stringify!{ henceforth })
- }
- }
- }).to_string());
-
- let maybe_happened = Field::from(&fields[1]);
- assert_eq!(maybe_happened.writer_snippet().to_string(),(quote!{
- {
- let definition_levels : Vec<i16> = self.iter().map(|rec| if rec.maybe_happened.is_some() { 1 } else { 0 }).collect();
- let vals : Vec<_> = records.iter().filter_map(|rec| {
- rec.maybe_happened.map(|inner| { inner.timestamp_millis() })
- }).collect();
-
- if let parquet::column::writer::ColumnWriter::Int64ColumnWriter(ref mut typed) = column_writer {
- typed.write_batch(&vals[..], Some(&definition_levels[..]), None) ?;
- } else {
- panic!("Schema and struct disagree on type for {}" , stringify!{ maybe_happened })
- }
- }
- }).to_string());
- }
-
- #[test]
- #[cfg(feature = "chrono")]
- fn test_chrono_date() {
- let snippet: proc_macro2::TokenStream = quote! {
- struct ATimestampStruct {
- henceforth: chrono::NaiveDate,
- maybe_happened: Option<&chrono::NaiveDate>,
- }
- };
-
- let fields = extract_fields(snippet);
- let when = Field::from(&fields[0]);
- assert_eq!(when.writer_snippet().to_string(),(quote!{
- {
- let vals : Vec<_> = records.iter().map(|rec| rec.henceforth.signed_duration_since(chrono::NaiveDate::from_ymd(1970, 1, 1)).num_days() as i32).collect();
- if let parquet::column::writer::ColumnWriter::Int32ColumnWriter(ref mut typed) = column_writer {
- typed.write_batch(&vals[..], None, None) ?;
- } else {
- panic!("Schema and struct disagree on type for {}" , stringify!{ henceforth })
- }
- }
- }).to_string());
-
- let maybe_happened = Field::from(&fields[1]);
- assert_eq!(maybe_happened.writer_snippet().to_string(),(quote!{
- {
- let definition_levels : Vec<i16> = self.iter().map(|rec| if rec.maybe_happened.is_some() { 1 } else { 0 }).collect();
- let vals : Vec<_> = records.iter().filter_map(|rec| {
- rec.maybe_happened.map(|inner| { inner.signed_duration_since(chrono::NaiveDate::from_ymd(1970, 1, 1)).num_days() as i32 })
- }).collect();
-
- if let parquet::column::writer::ColumnWriter::Int32ColumnWriter(ref mut typed) = column_writer {
- typed.write_batch(&vals[..], Some(&definition_levels[..]), None) ?;
- } else {
- panic!("Schema and struct disagree on type for {}" , stringify!{ maybe_happened })
- }
- }
- }).to_string());
- }
-
- #[test]
- #[cfg(feature = "uuid")]
- fn test_uuid() {
- let snippet: proc_macro2::TokenStream = quote! {
- struct ATimestampStruct {
- unique_id: uuid::Uuid,
- maybe_unique_id: Option<&uuid::Uuid>,
- }
- };
-
- let fields = extract_fields(snippet);
- let when = Field::from(&fields[0]);
- assert_eq!(when.writer_snippet().to_string(),(quote!{
- {
- let vals : Vec<_> = records.iter().map(|rec| (&rec.unique_id.to_string()[..]).into() ).collect();
- if let parquet::column::writer::ColumnWriter::ByteArrayColumnWriter(ref mut typed) = column_writer {
- typed.write_batch(&vals[..], None, None) ?;
- } else {
- panic!("Schema and struct disagree on type for {}" , stringify!{ unique_id })
- }
- }
- }).to_string());
-
- let maybe_happened = Field::from(&fields[1]);
- assert_eq!(maybe_happened.writer_snippet().to_string(),(quote!{
- {
- let definition_levels : Vec<i16> = self.iter().map(|rec| if rec.maybe_unique_id.is_some() { 1 } else { 0 }).collect();
- let vals : Vec<_> = records.iter().filter_map(|rec| {
- rec.maybe_unique_id.map(|ref inner| { (&inner.to_string()[..]).into() })
- }).collect();
-
- if let parquet::column::writer::ColumnWriter::ByteArrayColumnWriter(ref mut typed) = column_writer {
- typed.write_batch(&vals[..], Some(&definition_levels[..]), None) ?;
- } else {
- panic!("Schema and struct disagree on type for {}" , stringify!{ maybe_unique_id })
- }
- }
- }).to_string());
- }
-}
diff --git a/parquet_derive_test/Cargo.toml b/parquet_derive_test/Cargo.toml
deleted file mode 100644
index 25c814b..0000000
--- a/parquet_derive_test/Cargo.toml
+++ /dev/null
@@ -1,32 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-[package]
-name = "parquet_derive_test"
-version = "5.0.0-SNAPSHOT"
-license = "Apache-2.0"
-description = "Integration test package for parquet-derive"
-homepage = "https://github.com/apache/arrow-rs"
-repository = "https://github.com/apache/arrow-rs"
-authors = ["Apache Arrow <dev@arrow.apache.org>"]
-keywords = [ "parquet" ]
-edition = "2018"
-publish = false
-
-[dependencies]
-parquet = { path = "../parquet", version = "5.0.0-SNAPSHOT" }
-parquet_derive = { path = "../parquet_derive", version = "5.0.0-SNAPSHOT" }
diff --git a/parquet_derive_test/src/lib.rs b/parquet_derive_test/src/lib.rs
deleted file mode 100644
index b4bfc42..0000000
--- a/parquet_derive_test/src/lib.rs
+++ /dev/null
@@ -1,131 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#![allow(clippy::approx_constant)]
-
-extern crate parquet;
-
-#[macro_use]
-extern crate parquet_derive;
-
-use parquet::record::RecordWriter;
-
-#[derive(ParquetRecordWriter)]
-struct ACompleteRecord<'a> {
- pub a_bool: bool,
- pub a_str: &'a str,
- pub a_string: String,
- pub a_borrowed_string: &'a String,
- pub maybe_a_str: Option<&'a str>,
- pub maybe_a_string: Option<String>,
- pub magic_number: i32,
- pub low_quality_pi: f32,
- pub high_quality_pi: f64,
- pub maybe_pi: Option<f32>,
- pub maybe_best_pi: Option<f64>,
- pub borrowed_maybe_a_string: &'a Option<String>,
- pub borrowed_maybe_a_str: &'a Option<&'a str>,
-}
-
-#[cfg(test)]
-mod tests {
- use super::*;
-
- use parquet::{
- file::{
- properties::WriterProperties,
- writer::{FileWriter, SerializedFileWriter},
- },
- schema::parser::parse_message_type,
- };
- use std::{env, fs, io::Write, sync::Arc};
-
- #[test]
- fn test_parquet_derive_hello() {
- let file = get_temp_file("test_parquet_derive_hello", &[]);
- let schema_str = "message schema {
- REQUIRED boolean a_bool;
- REQUIRED BINARY a_str (UTF8);
- REQUIRED BINARY a_string (UTF8);
- REQUIRED BINARY a_borrowed_string (UTF8);
- OPTIONAL BINARY a_maybe_str (UTF8);
- OPTIONAL BINARY a_maybe_string (UTF8);
- REQUIRED INT32 magic_number;
- REQUIRED FLOAT low_quality_pi;
- REQUIRED DOUBLE high_quality_pi;
- OPTIONAL FLOAT maybe_pi;
- OPTIONAL DOUBLE maybe_best_pi;
- OPTIONAL BINARY borrowed_maybe_a_string (UTF8);
- OPTIONAL BINARY borrowed_maybe_a_str (UTF8);
- }";
-
- let schema = Arc::new(parse_message_type(schema_str).unwrap());
-
- let props = Arc::new(WriterProperties::builder().build());
- let mut writer = SerializedFileWriter::new(file, schema, props).unwrap();
-
- let a_str = "hello mother".to_owned();
- let a_borrowed_string = "cool news".to_owned();
- let maybe_a_string = Some("it's true, I'm a string".to_owned());
- let maybe_a_str = Some(&a_str[..]);
-
- let drs: Vec<ACompleteRecord> = vec![ACompleteRecord {
- a_bool: true,
- a_str: &a_str[..],
- a_string: "hello father".into(),
- a_borrowed_string: &a_borrowed_string,
- maybe_a_str: Some(&a_str[..]),
- maybe_a_string: Some(a_str.clone()),
- magic_number: 100,
- low_quality_pi: 3.14,
- high_quality_pi: 3.1415,
- maybe_pi: Some(3.14),
- maybe_best_pi: Some(3.1415),
- borrowed_maybe_a_string: &maybe_a_string,
- borrowed_maybe_a_str: &maybe_a_str,
- }];
-
- let mut row_group = writer.next_row_group().unwrap();
- drs.as_slice().write_to_row_group(&mut row_group).unwrap();
- writer.close_row_group(row_group).unwrap();
- writer.close().unwrap();
- }
-
- /// Returns file handle for a temp file in 'target' directory with a provided content
- pub fn get_temp_file(file_name: &str, content: &[u8]) -> fs::File {
- // build tmp path to a file in "target/debug/testdata"
- let mut path_buf = env::current_dir().unwrap();
- path_buf.push("target");
- path_buf.push("debug");
- path_buf.push("testdata");
- fs::create_dir_all(&path_buf).unwrap();
- path_buf.push(file_name);
-
- // write file content
- let mut tmp_file = fs::File::create(path_buf.as_path()).unwrap();
- tmp_file.write_all(content).unwrap();
- tmp_file.sync_all().unwrap();
-
- // return file handle for both read and write
- let file = fs::OpenOptions::new()
- .read(true)
- .write(true)
- .open(path_buf.as_path());
- assert!(file.is_ok());
- file.unwrap()
- }
-}
diff --git a/pre-commit.sh b/pre-commit.sh
deleted file mode 100755
index 5ce0807..0000000
--- a/pre-commit.sh
+++ /dev/null
@@ -1,88 +0,0 @@
-#!/bin/bash
-
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-# This file is git pre-commit hook.
-#
-# Soft link it as git hook under top dir of apache arrow git repository:
-# $ ln -s ../../rust/pre-commit.sh .git/hooks/pre-commit
-#
-# This file be run directly:
-# $ ./pre-commit.sh
-
-function RED() {
- echo "\033[0;31m$@\033[0m"
-}
-
-function GREEN() {
- echo "\033[0;32m$@\033[0m"
-}
-
-function BYELLOW() {
- echo "\033[1;33m$@\033[0m"
-}
-
-RUST_DIR="rust"
-
-# env GIT_DIR is set by git when run a pre-commit hook.
-if [ -z "${GIT_DIR}" ]; then
- GIT_DIR=$(git rev-parse --show-toplevel)
-fi
-
-cd ${GIT_DIR}/${RUST_DIR}
-
-NUM_CHANGES=$(git diff --cached --name-only . |
- grep -e ".*/*.rs$" |
- awk '{print $1}' |
- wc -l)
-
-if [ ${NUM_CHANGES} -eq 0 ]; then
- echo -e "$(GREEN INFO): no staged changes in *.rs, $(GREEN skip cargo fmt/clippy)"
- exit 0
-fi
-
-# 1. cargo clippy
-
-echo -e "$(GREEN INFO): cargo clippy ..."
-
-# Cargo clippy always return exit code 0, and `tee` doesn't work.
-# So let's just run cargo clippy.
-cargo clippy
-echo -e "$(GREEN INFO): cargo clippy done"
-
-# 2. cargo fmt: format with nightly and stable.
-
-CHANGED_BY_CARGO_FMT=false
-echo -e "$(GREEN INFO): cargo fmt with nightly and stable ..."
-
-for version in nightly stable; do
- CMD="cargo +${version} fmt"
- ${CMD} --all -q -- --check 2>/dev/null
- if [ $? -ne 0 ]; then
- ${CMD} --all
- echo -e "$(BYELLOW WARN): ${CMD} changed some files"
- CHANGED_BY_CARGO_FMT=true
- fi
-done
-
-if ${CHANGED_BY_CARGO_FMT}; then
- echo -e "$(RED FAIL): git commit $(RED ABORTED), please have a look and run git add/commit again"
- exit 1
-fi
-
-exit 0
diff --git a/testing b/testing
deleted file mode 160000
index b658b08..0000000
--- a/testing
+++ /dev/null
@@ -1 +0,0 @@
-Subproject commit b658b087767b041b2081766814655b4dd5a9a439
