chore: prepare Apache Paimon C++ 0.2.3 release (#155)
* docs: align feature guidance with implementation
* docs: clarify build and commit integration
* chore: prepare Apache Paimon C++ 0.2.3 release
diff --git a/.github/workflows/build_release.yaml b/.github/workflows/build_release.yaml
index fe25c7a..0bed7ce 100644
--- a/.github/workflows/build_release.yaml
+++ b/.github/workflows/build_release.yaml
@@ -41,8 +41,6 @@
steps:
- name: Checkout paimon-cpp
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
- with:
- lfs: true
- name: Setup ccache
uses: ./.github/actions/setup-ccache
with:
@@ -64,8 +62,6 @@
steps:
- name: Checkout paimon-cpp
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
- with:
- lfs: true
- name: Setup ccache
uses: ./.github/actions/setup-ccache
with:
diff --git a/.github/workflows/clang_test.yaml b/.github/workflows/clang_test.yaml
index 20efbfa..ea30689 100644
--- a/.github/workflows/clang_test.yaml
+++ b/.github/workflows/clang_test.yaml
@@ -42,7 +42,6 @@
- name: Checkout paimon-cpp
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
with:
- lfs: true
fetch-depth: 0 # fetch all history for git diff in clang-tidy
- name: Setup ccache
uses: ./.github/actions/setup-ccache
diff --git a/.github/workflows/gcc8_test.yaml b/.github/workflows/gcc8_test.yaml
index 405ffb0..01ed551 100644
--- a/.github/workflows/gcc8_test.yaml
+++ b/.github/workflows/gcc8_test.yaml
@@ -45,7 +45,7 @@
- name: Install dependencies
run: |
apt-get update
- DEBIAN_FRONTEND=noninteractive apt-get install -y gcc-8 g++-8 ninja-build git git-lfs tar curl tzdata zip unzip pkg-config build-essential python3-dev gdb sudo
+ DEBIAN_FRONTEND=noninteractive apt-get install -y gcc-8 g++-8 ninja-build git tar curl tzdata zip unzip pkg-config build-essential python3-dev gdb sudo
curl -L -O https://github.com/Kitware/CMake/releases/download/v3.28.3/cmake-3.28.3-linux-x86_64.tar.gz
tar -zxvf cmake-3.28.3-linux-x86_64.tar.gz -C /usr/local --strip-components=1
rm cmake-3.28.3-linux-x86_64.tar.gz
@@ -60,8 +60,6 @@
ls -la
- name: Checkout paimon-cpp
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
- with:
- lfs: true
- name: Setup ccache
uses: ./.github/actions/setup-ccache
with:
diff --git a/.github/workflows/gcc_test.yaml b/.github/workflows/gcc_test.yaml
index 8d23c7e..b213f66 100644
--- a/.github/workflows/gcc_test.yaml
+++ b/.github/workflows/gcc_test.yaml
@@ -41,8 +41,6 @@
steps:
- name: Checkout paimon-cpp
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
- with:
- lfs: true
- name: Setup ccache
uses: ./.github/actions/setup-ccache
with:
diff --git a/.github/workflows/test_with_sanitizer.yaml b/.github/workflows/test_with_sanitizer.yaml
index 06436eb..4005c8a 100644
--- a/.github/workflows/test_with_sanitizer.yaml
+++ b/.github/workflows/test_with_sanitizer.yaml
@@ -41,8 +41,6 @@
steps:
- name: Checkout paimon-cpp
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
- with:
- lfs: true
- name: Setup ccache
uses: ./.github/actions/setup-ccache
with:
diff --git a/.gitignore b/.gitignore
index a700cb5..b71dc56 100644
--- a/.gitignore
+++ b/.gitignore
@@ -20,6 +20,7 @@
build-release
build-debug
output
+release
# IDE settings
.idea
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 1726103..b6aca77 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -37,7 +37,7 @@
endif()
project(paimon
- VERSION 0.2.0
+ VERSION 0.2.3
DESCRIPTION "Paimon C++ Project")
string(TOUPPER "${CMAKE_BUILD_TYPE}" UPPERCASE_BUILD_TYPE)
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 5aa4531..086f4a6 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -59,7 +59,6 @@
- **C++17** compatible compiler (GCC recommended)
- **CMake** ≥ 3.16
- **Python 3** (for linting scripts and pre-commit)
-- **git-lfs** (the repository uses Git LFS for large files)
---
diff --git a/README.md b/README.md
index 44b926e..57e32d4 100644
--- a/README.md
+++ b/README.md
@@ -49,14 +49,9 @@
## Building
-> **Note:** The build system and source files are being migrated incrementally. The instructions below will work once the CMake build files and source code are available in this repository.
-
-If you do not have `git-lfs` installed, install it first.
-
```bash
git clone https://github.com/apache/paimon-cpp.git
cd paimon-cpp
-git lfs pull
```
Build with CMake:
@@ -80,7 +75,7 @@
## Collaboration
Paimon-cpp is an active open-source project and we welcome people who want to contribute or share good ideas!
-Before contributing, please read the [Contributing Guide](CONTRIBUTING.md) and the [Code Style Guide](docs/code-style.md). You are encouraged to check out our [documentation](https://alibaba.github.io/paimon-cpp/).
+Before contributing, please read the [Contributing Guide](CONTRIBUTING.md) and the [Code Style Guide](docs/code-style.md). You are encouraged to check out our [documentation](https://paimon.apache.org/docs/cpp/).
## License
diff --git a/docs/source/_static/versions.json b/docs/source/_static/versions.json
index 9980db1..28cf190 100644
--- a/docs/source/_static/versions.json
+++ b/docs/source/_static/versions.json
@@ -1,7 +1,7 @@
[
{
- "name": "0.10.0 (dev)",
- "version": "dev/",
- "url": "#"
+ "name": "0.2.3",
+ "version": "0.2.3",
+ "url": "https://paimon.apache.org/docs/cpp/"
}
]
diff --git a/docs/source/api/file_format.rst b/docs/source/api/file_format.rst
index 57b9eee..23c88cc 100644
--- a/docs/source/api/file_format.rst
+++ b/docs/source/api/file_format.rst
@@ -36,10 +36,6 @@
:members:
:undoc-members:
-.. doxygenclass:: paimon::BatchReader
- :members:
- :undoc-members:
-
.. doxygenclass:: paimon::FileBatchReader
:members:
:undoc-members:
diff --git a/docs/source/basic_concepts.rst b/docs/source/basic_concepts.rst
index 0cf87f4..84b8b5e 100644
--- a/docs/source/basic_concepts.rst
+++ b/docs/source/basic_concepts.rst
@@ -54,11 +54,8 @@
Data Files
---------------------------
-Data files are grouped by partitions. Currently, Paimon supports using parquet
-(default) and orc as data file’s format.
-
-.. note::
- avro write as a data file format is not supported yet.
+Data files are grouped by partitions. Paimon C++ supports Parquet (default),
+ORC, and Avro as data file formats.
Partition
---------------------------
@@ -84,6 +81,3 @@
modify the same partition, their commits can occur in parallel. If they modify
the same partition, only snapshot isolation is guaranteed. That is, the final
table state may be a mix of the two commits, but no changes are lost.
-
-.. note::
- Paimon C++ currently does not support compaction.
diff --git a/docs/source/build_system.rst b/docs/source/build_system.rst
index 8bcf714..f3b18e5 100644
--- a/docs/source/build_system.rst
+++ b/docs/source/build_system.rst
@@ -29,7 +29,7 @@
and available to your build system
The recommended way to integrate the Paimon C++ libraries into your C++ project
-is to use CMake’s `find_package <https://cmake.org/cmake/help/latest/command/find_package.html>`_
+is to use CMake's `find_package <https://cmake.org/cmake/help/latest/command/find_package.html>`_
function to locate and integrate dependencies.
CMake
@@ -48,11 +48,27 @@
project(MyExample)
- find_package(Arrow REQUIRED)
- find_package(Paimon REQUIRED)
+ # Arrow's static package may export LZ4 under a different target name.
+ find_package(lz4 CONFIG QUIET)
+ if(TARGET LZ4::lz4_static AND NOT TARGET LZ4::lz4)
+ add_library(LZ4::lz4 ALIAS LZ4::lz4_static)
+ elseif(TARGET lz4::lz4 AND NOT TARGET LZ4::lz4)
+ add_library(LZ4::lz4 ALIAS lz4::lz4)
+ endif()
+
+ find_package(Arrow CONFIG REQUIRED)
+ find_package(Paimon CONFIG REQUIRED)
+
+ if(TARGET Arrow::arrow_shared)
+ set(PAIMON_ARROW_TARGET Arrow::arrow_shared)
+ elseif(TARGET Arrow::arrow_static)
+ set(PAIMON_ARROW_TARGET Arrow::arrow_static)
+ else()
+ message(FATAL_ERROR "No supported Arrow CMake target is available")
+ endif()
add_executable(my_example my_example.cc)
- target_link_libraries(my_example PRIVATE Arrow::arrow_shared
+ target_link_libraries(my_example PRIVATE ${PAIMON_ARROW_TARGET}
Paimon::paimon_shared
Paimon::paimon_parquet_file_format_shared
Paimon::paimon_local_file_system_shared)
@@ -64,12 +80,15 @@
Paimon C++ installation on your system. If successful, it sets ``Paimon_FOUND``
to true if the Paimon C++ libraries were found.
-It also defines the following linkable imported targets:
+It defines the following supported imported target:
* ``Paimon::paimon_shared`` links to the Paimon shared libraries
-* ``Paimon::paimon_static`` links to the Paimon static libraries
-In most cases, it is recommended to use the Paimon shared libraries.
+Static targets such as ``Paimon::paimon_static`` may also be present when
+Paimon is built with ``PAIMON_BUILD_STATIC=ON``. They are not currently
+supported as standalone installed targets because their exported link
+interfaces do not include every third-party dependency. Installed consumers
+should use the shared targets.
Optional plugins (built-in file formats, file systems, and index)
-----------------------------------------------------------------
@@ -78,22 +97,24 @@
- File format plugins:
- - ``Paimon::paimon_parquet_file_format_shared`` / ``Paimon::paimon_parquet_file_format_static``
- - ``Paimon::paimon_orc_file_format_shared`` / ``Paimon::paimon_orc_file_format_static``
- - ``Paimon::paimon_avro_file_format_shared`` / ``Paimon::paimon_avro_file_format_static``
- - ``Paimon::paimon_blob_file_format_shared`` / ``Paimon::paimon_blob_file_format_static``
+ - ``Paimon::paimon_parquet_file_format_shared``
+ - ``Paimon::paimon_orc_file_format_shared``
+ - ``Paimon::paimon_avro_file_format_shared``
+ - ``Paimon::paimon_blob_file_format_shared``
- File system plugins:
- - ``Paimon::paimon_local_file_system_shared`` / ``Paimon::paimon_local_file_system_static``
- - ``Paimon::paimon_jindo_file_system_shared`` / ``Paimon::paimon_jindo_file_system_static``
+ - ``Paimon::paimon_local_file_system_shared``
+ - ``Paimon::paimon_jindo_file_system_shared``
- Index plugins:
- - ``Paimon::paimon_file_index_shared`` / ``Paimon::paimon_file_index_static``
- - ``Paimon::paimon_lumina_index_shared`` / ``Paimon::paimon_lumina_index_static``
- - ``Paimon::paimon_lucene_index_shared`` / ``Paimon::paimon_lucene_index_static``
+ - ``Paimon::paimon_file_index_shared``
+ - ``Paimon::paimon_lumina_index_shared``
+ - ``Paimon::paimon_lucene_index_shared``
.. note::
- In most cases, it is recommended to use the shared variants of these plugins.
+ Static plugin targets have the same installed-consumer limitation as
+ ``Paimon::paimon_static`` and should not be used as standalone imported
+ targets.
diff --git a/docs/source/building.rst b/docs/source/building.rst
index 71df728..8b106c0 100644
--- a/docs/source/building.rst
+++ b/docs/source/building.rst
@@ -60,7 +60,6 @@
$ git clone https://github.com/apache/paimon-cpp.git
$ cd paimon-cpp
- $ git lfs pull
Manual configuration
--------------------
@@ -160,14 +159,22 @@
-Dzstd_SOURCE=BUNDLED
Use ``PAIMON_PACKAGE_PREFIX`` to provide one common prefix for dependencies
-whose own ``<Package>_ROOT`` variable is not set.
+whose own ``<Package>_ROOT`` variable is not set. Because the patched Arrow and
+ORC dependencies cannot be resolved from the system, a global ``SYSTEM`` build
+must override them to ``BUNDLED``:
.. code-block:: shell
cmake -B build \
-DPAIMON_DEPENDENCY_SOURCE=SYSTEM \
+ -DArrow_SOURCE=BUNDLED \
+ -DORC_SOURCE=BUNDLED \
-DPAIMON_PACKAGE_PREFIX=/opt/paimon-deps
+All other enabled dependencies must be available as system packages or under
+the specified prefix. When ORC support is disabled, the ``ORC_SOURCE`` override
+can be omitted.
+
Package-manager-specific modes are intentionally out of scope for this first
dependency source interface. They can still be used through standard CMake
mechanisms such as ``CMAKE_PREFIX_PATH`` or ``CMAKE_TOOLCHAIN_FILE``, while
diff --git a/docs/source/conf.py b/docs/source/conf.py
index 5789bbc..989ea07 100644
--- a/docs/source/conf.py
+++ b/docs/source/conf.py
@@ -114,7 +114,7 @@
# The master toctree document.
master_doc = "index"
-version = "0.2.0"
+version = "0.2.3"
html_theme_options = {
"show_toc_level": 2,
diff --git a/docs/source/index.rst b/docs/source/index.rst
index c595d45..5c4d83a 100644
--- a/docs/source/index.rst
+++ b/docs/source/index.rst
@@ -86,7 +86,7 @@
:class-card: contrib-card
:shadow: none
- Explore Paimon‘s API reference documentation
+ Explore Paimon's API reference documentation
+++
diff --git a/docs/source/user_guide/arrow.rst b/docs/source/user_guide/arrow.rst
index 084b156..a2e0229 100644
--- a/docs/source/user_guide/arrow.rst
+++ b/docs/source/user_guide/arrow.rst
@@ -62,7 +62,7 @@
Adopting the Arrow C Data Interface
-----------------------------------
-To leverage Arrow’s performance and ecosystem benefits while avoiding tight
+To leverage Arrow's performance and ecosystem benefits while avoiding tight
coupling to specific Arrow C++ SDK versions, we use the Arrow C Data
Interface as the default in-memory format for Paimon C++.
@@ -114,7 +114,7 @@
Nullability and validity
~~~~~~~~~~~~~~~~~~~~~~~~
-- Use standard validity bitmaps for nullability and adhere to Arrow’s canonical buffer organization (validity, offsets, data, etc.).
+- Use standard validity bitmaps for nullability and adhere to Arrow's canonical buffer organization (validity, offsets, data, etc.).
Conversion boundaries
~~~~~~~~~~~~~~~~~~~~~~
diff --git a/docs/source/user_guide/clean.rst b/docs/source/user_guide/clean.rst
index 05e2645..32099dd 100644
--- a/docs/source/user_guide/clean.rst
+++ b/docs/source/user_guide/clean.rst
@@ -85,7 +85,7 @@
1. Build a ``ScanFilter`` for the partition and use the latest snapshot
to scan the partition.
2. Iterate over the scanned data file list (``ManifestEntries``) and
- rewrite each entry’s type to ``DELETE``.
+ rewrite each entry's type to ``DELETE``.
3. Commit using the rewritten ``ManifestEntries``. If the commit fails,
retry a limited number of times.
diff --git a/docs/source/user_guide/commit.rst b/docs/source/user_guide/commit.rst
index e833991..7e7ee17 100644
--- a/docs/source/user_guide/commit.rst
+++ b/docs/source/user_guide/commit.rst
@@ -18,7 +18,7 @@
Commit
==========================================
-Commit is a critical stage in Paimon’s write path. It is responsible for generating
+Commit is a critical stage in Paimon's write path. It is responsible for generating
Snapshot files that describe the current state of a Paimon table. This document
provides a detailed analysis of the Paimon Commit process.
@@ -44,7 +44,7 @@
~~~~~~~~~~~~~~~~~~~~~~~~
During Commit, the system extracts key information from ``CommitMessages``—such as
-file name, operation type (``ADD`` or ``DELETE``), the file’s Partition and Bucket—
+file name, operation type (``ADD`` or ``DELETE``), the file's Partition and Bucket—
and converts them into ``ManifestEntry`` records.
A ``ManifestEntry`` represents a single operation record in a manifest file and
@@ -83,7 +83,7 @@
Minor Compaction
^^^^^^^^^^^^^^^^
-If Full Compaction’s conditions are not met, Minor Compaction is attempted:
+If Full Compaction's conditions are not met, Minor Compaction is attempted:
- The system iterates over all files, skipping any file larger than the target file size.
- Whenever the accumulated size of selected files exceeds the target file size, those files are merged.
@@ -101,7 +101,7 @@
After compaction (which may or may not be triggered every time), the system obtains
a consolidated set of manifest file metadata. This metadata is written into a new
-manifest list file, forming the Snapshot’s base manifest list.
+manifest list file, forming the Snapshot's base manifest list.
Generate New Manifest Files and the Delta Manifest List
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
diff --git a/docs/source/user_guide/data_types.rst b/docs/source/user_guide/data_types.rst
index 5759119..de794d8 100644
--- a/docs/source/user_guide/data_types.rst
+++ b/docs/source/user_guide/data_types.rst
@@ -150,8 +150,9 @@
- Data type of a timestamp without time zone consisting of year-month-day hour:minute:second[.fractional] with up to nanosecond precision and values ranging from 0000-01-01 00:00:00.000000000 to 9999-12-31 23:59:59.999999999.
The type can be declared using ``TIMESTAMP(p)`` where p is the number of
- digits of fractional seconds (precision). p must have a value between 0
- and 9 (both inclusive). If no precision is specified, p is equal to 6.
+ digits of fractional seconds (precision). Paimon C++ supports precision
+ values 0 (seconds), 3 (milliseconds), 6 (microseconds), and 9
+ (nanoseconds). If no precision is specified, p is equal to 6.
* - ``TIMESTAMP WITH LOCAL TIME ZONE``
@@ -164,6 +165,9 @@
to the configured session time zone. A conversion from and to int describes
the number of seconds since epoch. A conversion from and to long describes the number of milliseconds since epoch.
+ Paimon C++ supports precision values 0 (seconds), 3 (milliseconds), 6
+ (microseconds), and 9 (nanoseconds).
+
* - ``ARRAY<t>``
- List
- Data type of an array of elements with same subtype.
diff --git a/docs/source/user_guide/prefetch.rst b/docs/source/user_guide/prefetch.rst
index 15ae9d7..9f62da2 100644
--- a/docs/source/user_guide/prefetch.rst
+++ b/docs/source/user_guide/prefetch.rst
@@ -23,26 +23,50 @@
:align: center
:width: 100%
-In C++ Paimon, we use a multi-producer, single-consumer model to optimize file reading. The core idea is to split a file into line-based ReadRanges and assign them to multiple reader threads (producers). Each reader thread owns an independent result queue that holds its processed RecordBatches. In the main reader thread (the consumer), we sort the heads of all queues by the ReadRange start offset in ascending order and select the RecordBatch with the smallest start offset to ensure globally ordered results.
+In C++ Paimon, we use a multi-producer, single-consumer model to optimize file
+reading. The core idea is to split a file into line-based ReadRanges and assign
+them to multiple reader threads (producers). Each reader thread owns an
+independent result queue that holds its processed RecordBatches. In the main
+reader thread (the consumer), we sort the heads of all queues by the ReadRange
+start offset in ascending order and select the RecordBatch with the smallest
+start offset to ensure globally ordered results.
Read Range Splitting Strategy
=============================
-Designing an efficient ReadRange splitting strategy requires balancing two key objectives:
+Designing an efficient ReadRange splitting strategy requires balancing two key
+objectives:
-- Minimize read amplification: Ensure the data fetched from storage is used effectively, avoiding unnecessary I/O overhead.
-- Reduce ReadRange span: Ideally, the size of a ReadRange should match a single read batch size to enable fine-grained parallel control.
+- Minimize read amplification: Ensure the data fetched from storage is used
+ effectively, avoiding unnecessary I/O overhead.
+- Reduce ReadRange span: Ideally, the size of a ReadRange should match a single
+ read batch size to enable fine-grained parallel control.
-Below we detail how these strategies are applied to formats Parquet.
+Below we detail how these strategies are applied to Parquet and ORC.
Parquet
========
-Parquet files are organized into RowGroups and Pages. Since C++ Parquet does not support row-level seeking, prefetching can only be done at the RowGroup level. This naturally avoids read amplification, but introduces a new challenge: if a file contains only a small number of RowGroups, parallelism is severely limited. Therefore, we recommend users reduce RowGroup size when writing Parquet files to increase opportunities for parallel processing.
+Parquet files are organized into RowGroups and Pages. Since C++ Parquet does
+not support row-level seeking, prefetching can only be done at the RowGroup
+level. This naturally avoids read amplification, but introduces a new
+challenge: if a file contains only a small number of RowGroups, parallelism is
+severely limited. Therefore, we recommend users reduce RowGroup size when
+writing Parquet files to increase opportunities for parallel processing.
-Another critical difference is the read behavior compared to Orc. Orc strictly returns RecordBatches aligned to Stripe boundaries, whereas C++ Parquet may return a RecordBatch containing data from multiple RowGroups. This can lead to output order confusion during parallel reads. We modified C++ Parquet internals to return results strictly aligned to RowGroup boundaries, matching Orc’s behavior. With this change, parallel reading no longer requires complex seek operations, improving overall read efficiency.
+Another critical difference is the read behavior compared to ORC. ORC strictly
+returns RecordBatches aligned to Stripe boundaries, whereas C++ Parquet may
+return a RecordBatch containing data from multiple RowGroups. This can lead to
+output order confusion during parallel reads. We modified C++ Parquet internals
+to return results strictly aligned to RowGroup boundaries, matching ORC's
+behavior. With this change, parallel reading no longer requires complex seek
+operations, improving overall read efficiency.
-.. admonition:: TODO
- :class: tip
+ORC
+===
- Support prefetch for Orc.
+ORC files are organized into Stripes. Paimon C++ generates row ranges from
+Stripe metadata, the selected columns, and the configured natural read size.
+Prefetch is enabled when the estimated compressed size of the selected columns
+exceeds the configured threshold; otherwise the reader uses the normal
+single-reader path.
diff --git a/docs/source/user_guide/write.rst b/docs/source/user_guide/write.rst
index da2120b..6fd5c41 100644
--- a/docs/source/user_guide/write.rst
+++ b/docs/source/user_guide/write.rst
@@ -17,14 +17,14 @@
Write
=====
-Batch writing requires the compute engine to pre-bucket data (bucket), using the
-same bucketing strategy as Paimon to ensure correct ``Scan`` behavior, and to
-specify the target ``partition``. Data should be accumulated into ``RecordBatch``
-and written to Paimon.
+Batch writing requires the compute engine to specify the target ``partition``.
+For fixed-bucket tables, the engine must also assign a valid bucket to every
+``RecordBatch``. In unaware-bucket and postpone-bucket modes, the writer can
+resolve the bucket automatically when it is omitted.
Paimon C++ uses Apache Arrow as the :ref:`in-memory columnar format<memory-format>`
-to more efficiently support writing to disk columnar formats such as ORC and
-Parquet, thereby improving write throughput.
+to more efficiently support writing to disk columnar formats such as ORC,
+Parquet, and Avro, thereby improving write throughput.
.. note::
Currently supported table types:
@@ -33,14 +33,13 @@
Not supported in the current scope:
- Changelog
- - Indexes
Bucketing Modes
---------------
- Append tables:
- * Support ``bucket = -1`` (dynamic bucket mode)
+ * Support ``bucket = -1`` (unaware-bucket mode)
* Support ``bucket > 0`` (fixed bucket mode)
- PK tables:
@@ -56,9 +55,15 @@
- The compute engine must:
- - Apply the Paimon-consistent bucketing function to each row prior to batching.
- Assign the correct ``partition`` for each row.
- - Group rows into Arrow ``RecordBatch`` per partition-bucket combination to minimize writer state changes and I/O overhead.
+ - In fixed-bucket mode, apply the Paimon-consistent bucketing function, set a
+ bucket in ``[0, bucket)``, and group rows into Arrow ``RecordBatch`` objects
+ per partition-bucket combination.
+
+- In unaware-bucket mode (append table with ``bucket = -1``), an omitted bucket
+ is resolved to ``0``.
+- In postpone-bucket mode (primary-key table with ``bucket = -2``), an omitted
+ bucket is resolved to ``-2``.
- Recommended practices:
@@ -70,7 +75,7 @@
----------------
The compute engine is responsible for triggering the writer nodes' ``PrepareCommit``.
-Triggering conditions depend on the engine’s business needs and can follow either:
+Triggering conditions depend on the engine's business needs and can follow either:
- Streaming mode: time-based or periodic triggers (e.g., every N seconds).
- Batch mode: trigger after all data in the batch has been written.
@@ -98,7 +103,7 @@
representations, Paimon C++ provides ``CommitMessage`` only through:
- Serialization: convert the internal commit state into a well-defined binary
- representation that matches Java Paimon’s expectations.
+ representation that matches Java Paimon's expectations.
- Deserialization: parse the Java-compatible binary representation back into
C++ commit structures for validation, replay, or tooling needs.
@@ -122,26 +127,27 @@
.. note::
- Current C++ scope supports Append and PK tables. Changelog is out of
- scope and should not be emitted in ``CommitMessage`` until
- explicitly supported.
+ The C++ writer supports Append and PK tables and can produce
+ ``CommitMessage`` objects for both. ``FileStoreCommit`` currently executes
+ local commits only for append-only tables on non-object-store file systems.
+ PK and object-store commit messages must be sent to an external control
+ plane. Changelog is out of scope and should not be emitted in
+ ``CommitMessage`` until explicitly supported.
Serialization and Deserialization
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-- Binary Format:
- - The binary payload must strictly conform to Java Paimon’s ``CommitMessage`` encoding.
- - Version tags or schema identifiers should be included to enable forwards/backwards compatibility and safe upgrades.
-
-- Serialization API:
- - Provide a function to serialize the writer’s commit state into a byte buffer (or stream) consumable by Java Paimon.
-
-- Deserialization API:
- - Provide a function to parse a Java-produced ``CommitMessage`` binary payload back into C++ commit structures for verification, replay, and testing.
-
-- Validation:
- - Include conformance tests to assert that C++ serialized payloads are accepted by Java Paimon.
- - Include round-trip tests to ensure C++ can parse Java-produced payloads and vice versa for supported message versions.
+- **Binary format:** The binary payload must strictly conform to Java Paimon's
+ ``CommitMessage`` encoding. It does not contain a version tag, so callers
+ must transport ``CommitMessage::CurrentVersion()`` separately and supply it
+ when deserializing.
+- **Serialization API:** Use ``CommitMessage::Serialize`` for one message or
+ ``CommitMessage::SerializeList`` for a list.
+- **Deserialization API:** Use ``CommitMessage::Deserialize`` or
+ ``CommitMessage::DeserializeList`` with the separately supplied
+ serialization version.
+- **Validation:** Conformance and round-trip tests must verify compatibility
+ with Java Paimon for supported message versions.
Operational Flow
~~~~~~~~~~~~~~~~~~~~~~~
@@ -155,12 +161,15 @@
3. Each writer invokes ``PrepareCommit``, which:
- Aggregates per-writer state into a ``CommitMessage``.
- - Serializes the message into a Java-compatible binary payload.
+ - Returns ``CommitMessage`` objects; it does not serialize them.
-4. The compute engine gathers ``CommitMessages`` from all writers.
+4. The compute engine gathers ``CommitMessage`` objects from all writers. For
+ cross-process transport, it explicitly calls ``Serialize`` or
+ ``SerializeList`` and carries ``CurrentVersion()`` alongside the payload.
-5. The compute engine issues a ``Commit`` request to the control plane with the
- collected messages, resulting in a new ``Snapshot``.
+5. For a supported local append-table commit, the engine passes the objects to
+ ``FileStoreCommit``. For PK tables or object-store paths, it sends the
+ serialized payload and version to an external control plane.
-6. The coordinator validates the messages, updates manifests/metadata, and
- finalizes the snapshot atomically.
+6. The local committer or external coordinator validates the messages, updates
+ manifests/metadata, and finalizes the snapshot atomically.
diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt
index 4fe7a8b..385b482 100644
--- a/examples/CMakeLists.txt
+++ b/examples/CMakeLists.txt
@@ -20,23 +20,40 @@
set(CMAKE_CXX_STANDARD 17)
+# Arrow's static export refers to LZ4::lz4, while some lz4 packages export a
+# differently named target. Define a compatibility alias before loading Arrow.
+find_package(lz4 CONFIG QUIET)
+if(TARGET LZ4::lz4_static AND NOT TARGET LZ4::lz4)
+ add_library(LZ4::lz4 ALIAS LZ4::lz4_static)
+elseif(TARGET lz4::lz4 AND NOT TARGET LZ4::lz4)
+ add_library(LZ4::lz4 ALIAS lz4::lz4)
+endif()
+
find_package(Arrow CONFIG REQUIRED)
find_package(Paimon CONFIG REQUIRED)
+if(TARGET Arrow::arrow_shared)
+ set(PAIMON_EXAMPLE_ARROW_TARGET Arrow::arrow_shared)
+elseif(TARGET Arrow::arrow_static)
+ set(PAIMON_EXAMPLE_ARROW_TARGET Arrow::arrow_static)
+else()
+ message(FATAL_ERROR "Neither Arrow::arrow_shared nor Arrow::arrow_static is available")
+endif()
+
add_executable(read_write_demo read_write_demo.cpp)
add_executable(clean_demo clean_demo.cpp)
set(CMAKE_EXE_LINKER_FLAGS "-Wl,--no-as-needed ${CMAKE_EXE_LINKER_FLAGS}")
target_link_libraries(read_write_demo
- PRIVATE Arrow::arrow_shared
+ PRIVATE ${PAIMON_EXAMPLE_ARROW_TARGET}
Paimon::paimon_shared
Paimon::paimon_parquet_file_format_shared
Paimon::paimon_orc_file_format_shared
Paimon::paimon_local_file_system_shared)
target_link_libraries(clean_demo
- PRIVATE Arrow::arrow_shared
+ PRIVATE ${PAIMON_EXAMPLE_ARROW_TARGET}
Paimon::paimon_shared
Paimon::paimon_parquet_file_format_shared
Paimon::paimon_orc_file_format_shared
diff --git a/examples/clean_demo.cpp b/examples/clean_demo.cpp
index d34cb83..b2888a8 100644
--- a/examples/clean_demo.cpp
+++ b/examples/clean_demo.cpp
@@ -106,17 +106,26 @@
std::string temp_table_path = std::string(argv[2]);
std::string clean_mode = std::string(argv[3]);
+ if (clean_mode != "orphan_file" && clean_mode != "drop_partition" &&
+ clean_mode != "expire_snapshot") {
+ std::cerr << "Unknown clean mode: " << clean_mode
+ << ". Expected orphan_file, drop_partition, or expire_snapshot." << std::endl;
+ return -1;
+ }
+
if (!CopyToTempDirectory(origin_table_path, temp_table_path)) {
return -1;
}
- std::map<std::string, std::string> clean_options;
paimon::Status status;
if (clean_mode == "orphan_file") {
std::cout << "enter the timestamp (ms) before which orphan files will be deleted"
<< std::endl;
- int64_t older_than_ms;
- std::cin >> older_than_ms;
+ int64_t older_than_ms = 0;
+ if (!(std::cin >> older_than_ms)) {
+ std::cerr << "Invalid timestamp: expected an integer in milliseconds." << std::endl;
+ return -1;
+ }
status = paimon::CleanOrphanFiles(temp_table_path, older_than_ms);
} else if (clean_mode == "drop_partition") {
std::cout << "enter partition key-value pairs to drop. type 'EOF EOF' to finish"
diff --git a/include/paimon/commit_message.h b/include/paimon/commit_message.h
index 3d8757a..4521892 100644
--- a/include/paimon/commit_message.h
+++ b/include/paimon/commit_message.h
@@ -32,8 +32,12 @@
class CommitMessageSerializer;
class MemoryPool;
-/// Commit message for partition and bucket. Support Serialize and Deserialize, compatible with java
-/// version.
+/// Commit message for partition and bucket. Supports serialization and deserialization compatible
+/// with the Java version.
+///
+/// @note Serialized payloads do not embed their serialization version. Transport
+/// `CurrentVersion()` alongside the payload and pass it explicitly to `Deserialize()` or
+/// `DeserializeList()`.
// TODO(yonghao.fyh): to add some statistics of write (e.g., write bytes)
class PAIMON_EXPORT CommitMessage {
public:
@@ -44,6 +48,7 @@
/// Serializes a single commit message to a binary string format.
/// The serialized format is compatible with the Java version of Paimon.
+ /// The serialization version is not included in the returned payload.
/// @param commit_message The commit message to serialize.
/// @param pool Memory pool for memory allocation during serialization.
/// @return Result containing the serialized string data, or an error if serialization fails.
@@ -51,6 +56,7 @@
const std::shared_ptr<MemoryPool>& pool);
/// Serializes a list of commit messages to a binary string format.
+ /// The serialization version is not included in the returned payload.
/// @param commit_messages Vector of commit messages to serialize.
/// @param pool Memory pool for memory allocation during serialization.
/// @return Result containing the serialized string data, or an error if serialization fails.
diff --git a/include/paimon/defs.h b/include/paimon/defs.h
index c15256c..c38cb49 100644
--- a/include/paimon/defs.h
+++ b/include/paimon/defs.h
@@ -92,8 +92,9 @@
static const char SEQUENCE_GROUP[];
/// @}
- /// "bucket" - Bucket number for file store. It should either be equal to -1 (dynamic bucket
- /// mode), or it must be greater than 0 (fixed bucket mode).
+ /// "bucket" - Bucket mode or bucket count for file store. Append tables support -1
+ /// (unaware-bucket mode), primary-key tables support -2 (postpone-bucket mode), and both
+ /// table types support values greater than 0 (fixed-bucket mode).
static const char BUCKET[];
/// "bucket-key" - Specify the paimon distribution policy. Data is assigned to each bucket
diff --git a/include/paimon/file_store_commit.h b/include/paimon/file_store_commit.h
index a702c41..66d3d39 100644
--- a/include/paimon/file_store_commit.h
+++ b/include/paimon/file_store_commit.h
@@ -43,6 +43,11 @@
///
/// The `FileStoreCommit` class provides interfaces for committing changes, expiring old snapshots,
/// dropping partitions, and retrieving commit metrics.
+///
+/// @note Local commit execution currently supports append-only tables on non-object-store file
+/// systems. `Create()` returns `NotImplemented` for primary-key tables and object-store paths.
+/// Primary-key writers can still produce `CommitMessage` objects; those messages must be committed
+/// by an external control plane.
class PAIMON_EXPORT FileStoreCommit {
public:
/// Create an instance of `FileStoreCommit`.
diff --git a/include/paimon/global_index/global_index_write_task.h b/include/paimon/global_index/global_index_write_task.h
index 6151382..a66f7ab 100644
--- a/include/paimon/global_index/global_index_write_task.h
+++ b/include/paimon/global_index/global_index_write_task.h
@@ -40,10 +40,10 @@
/// @param table_path Path to the table root directory where index files are stored.
/// @param field_name Name of the indexed column (must be present in the table schema).
/// @param index_type Type of global index to build (e.g., "bitmap", "lumina").
- /// @param index_split The indexed split containing the actual data (e.g., Parquet file) and
- // row id range [from, to] for data to build index.
+ /// @param indexed_split The indexed split containing the actual data (e.g., Parquet file) and
+ /// row id range [from, to] for data to build index.
/// The range must be fully contained within the data covered
- /// by the given `split`.
+ /// by the given `indexed_split`.
/// @param options Index-specific configuration (e.g., false positive rate for bloom
/// filters).
/// @param pool Memory pool for temporary allocations during index construction.
diff --git a/include/paimon/global_index/global_indexer_factory.h b/include/paimon/global_index/global_indexer_factory.h
index 6b0783a..e3d3069 100644
--- a/include/paimon/global_index/global_indexer_factory.h
+++ b/include/paimon/global_index/global_indexer_factory.h
@@ -52,7 +52,7 @@
static Result<std::unique_ptr<GlobalIndexer>> Get(
const std::string& identifier, const std::map<std::string, std::string>& options);
- /// Creates a `GlobalIndexer` using the current factory’s implementation and the given options.
+ /// Creates a `GlobalIndexer` using the current factory's implementation and the given options.
virtual Result<std::unique_ptr<GlobalIndexer>> Create(
const std::map<std::string, std::string>& options) const = 0;
};
diff --git a/scripts/releasing/README.md b/scripts/releasing/README.md
new file mode 100644
index 0000000..bf9b7dd
--- /dev/null
+++ b/scripts/releasing/README.md
@@ -0,0 +1,81 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+
+# Apache Paimon C++ release scripts
+
+These scripts create and verify the source artifact voted on by the Apache Paimon
+PMC. They do not publish artifacts, create tags, or move files between Apache
+distribution repositories.
+
+## Create a release candidate
+
+Create and push a signed RC tag before creating the source artifact:
+
+```bash
+git tag -s release-0.2.3-rc1 -m "Apache Paimon C++ 0.2.3 RC1"
+git push upstream release-0.2.3-rc1
+```
+
+Create the source artifact, SHA-512 checksum, and detached OpenPGP signature:
+
+```bash
+scripts/releasing/create_source_release.sh \
+ --version 0.2.3 \
+ --git-ref release-0.2.3-rc1 \
+ --output-dir release/0.2.3-rc1 \
+ --signing-key ASF_GPG_KEY_ID
+```
+
+The output files are:
+
+```text
+apache-paimon-cpp-0.2.3-src.tgz
+apache-paimon-cpp-0.2.3-src.tgz.asc
+apache-paimon-cpp-0.2.3-src.tgz.sha512
+```
+
+Existing files are never overwritten. A changed candidate must use a new RC
+directory and a new vote.
+
+## Verify a release candidate
+
+Download the source artifact, its `.asc` and `.sha512` files, and the Paimon
+`KEYS` file. Import `KEYS` into a temporary or dedicated GPG keyring, then run:
+
+```bash
+RAT_JAR=/path/to/apache-rat-0.16.1.jar \
+ scripts/releasing/verify_release_candidate.sh \
+ apache-paimon-cpp-0.2.3-src.tgz
+```
+
+The verifier checks:
+
+- the SHA-512 checksum and detached OpenPGP signature;
+- archive paths and the single `paimon-cpp-0.2.3/` root directory;
+- required `LICENSE`, `NOTICE`, build, and documentation files;
+- the CMake and documentation versions;
+- absence of common compiled artifact types;
+- Apache RAT results;
+- a release build and the test suite from the extracted source archive.
+
+The `--allow-unsigned` and `--skip-rat` options are only for local development
+of the release process. A voter may use `--skip-build` when the repository's
+Linux CI build command is not suitable for their platform, but must then build
+and test the extracted source distribution separately before casting a binding
+vote.
diff --git a/scripts/releasing/create_source_release.sh b/scripts/releasing/create_source_release.sh
new file mode 100755
index 0000000..fb0345a
--- /dev/null
+++ b/scripts/releasing/create_source_release.sh
@@ -0,0 +1,180 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -euo pipefail
+
+SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+SOURCE_ROOT=$(cd "${SCRIPT_DIR}/../.." && pwd)
+
+RELEASE_VERSION=""
+GIT_REF=""
+OUTPUT_DIR="${SOURCE_ROOT}/release"
+SIGNING_KEY=""
+
+usage() {
+ cat <<'EOF'
+Create an Apache Paimon C++ source release from an immutable Git ref.
+
+Usage:
+ create_source_release.sh --version VERSION --git-ref REF [options]
+
+Required:
+ --version VERSION Release version, for example 0.2.3
+ --git-ref REF Commit or signed RC tag to archive
+
+Options:
+ --output-dir DIR Output directory (default: <repository>/release)
+ --signing-key KEY_ID Create an ASCII-armored detached OpenPGP signature
+ -h, --help Show this help
+
+The script creates:
+ apache-paimon-cpp-VERSION-src.tgz
+ apache-paimon-cpp-VERSION-src.tgz.sha512
+ apache-paimon-cpp-VERSION-src.tgz.asc (when --signing-key is provided)
+
+Existing artifacts are never overwritten.
+EOF
+}
+
+fail() {
+ echo "Error: $*" >&2
+ exit 1
+}
+
+calculate_sha512() {
+ local file=$1
+ if command -v sha512sum >/dev/null 2>&1; then
+ sha512sum "${file}" | awk '{print $1}'
+ elif command -v shasum >/dev/null 2>&1; then
+ shasum -a 512 "${file}" | awk '{print $1}'
+ else
+ fail "sha512sum or shasum is required"
+ fi
+}
+
+while [[ $# -gt 0 ]]; do
+ case "$1" in
+ --version)
+ [[ $# -ge 2 ]] || fail "--version requires a value"
+ RELEASE_VERSION=$2
+ shift 2
+ ;;
+ --git-ref)
+ [[ $# -ge 2 ]] || fail "--git-ref requires a value"
+ GIT_REF=$2
+ shift 2
+ ;;
+ --output-dir)
+ [[ $# -ge 2 ]] || fail "--output-dir requires a value"
+ OUTPUT_DIR=$2
+ shift 2
+ ;;
+ --signing-key)
+ [[ $# -ge 2 ]] || fail "--signing-key requires a value"
+ SIGNING_KEY=$2
+ shift 2
+ ;;
+ -h|--help)
+ usage
+ exit 0
+ ;;
+ *)
+ fail "unknown argument: $1"
+ ;;
+ esac
+done
+
+[[ -n "${RELEASE_VERSION}" ]] || fail "--version is required"
+[[ -n "${GIT_REF}" ]] || fail "--git-ref is required"
+[[ "${RELEASE_VERSION}" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]] ||
+ fail "release version must use MAJOR.MINOR.PATCH format"
+
+git -C "${SOURCE_ROOT}" rev-parse --verify "${GIT_REF}^{commit}" >/dev/null 2>&1 ||
+ fail "Git ref does not resolve to a commit: ${GIT_REF}"
+
+for required_file in CMakeLists.txt LICENSE NOTICE; do
+ git -C "${SOURCE_ROOT}" cat-file -e "${GIT_REF}:${required_file}" 2>/dev/null ||
+ fail "${required_file} is missing from ${GIT_REF}"
+done
+
+CMAKE_VERSION=$(
+ git -C "${SOURCE_ROOT}" show "${GIT_REF}:CMakeLists.txt" |
+ sed -n 's/^[[:space:]]*VERSION[[:space:]]\+\([0-9][0-9.]*\).*$/\1/p' |
+ head -n 1
+)
+[[ "${CMAKE_VERSION}" == "${RELEASE_VERSION}" ]] ||
+ fail "CMake version ${CMAKE_VERSION:-<missing>} does not match ${RELEASE_VERSION}"
+
+DOCS_VERSION=$(
+ git -C "${SOURCE_ROOT}" show "${GIT_REF}:docs/source/conf.py" |
+ sed -n 's/^version = "\([^"]*\)"$/\1/p' |
+ head -n 1
+)
+[[ "${DOCS_VERSION}" == "${RELEASE_VERSION}" ]] ||
+ fail "documentation version ${DOCS_VERSION:-<missing>} does not match ${RELEASE_VERSION}"
+
+ARTIFACT_NAME="apache-paimon-cpp-${RELEASE_VERSION}-src.tgz"
+ARCHIVE_ROOT="paimon-cpp-${RELEASE_VERSION}"
+
+mkdir -p "${OUTPUT_DIR}"
+OUTPUT_DIR=$(cd "${OUTPUT_DIR}" && pwd)
+
+for suffix in "" ".sha512" ".asc"; do
+ [[ ! -e "${OUTPUT_DIR}/${ARTIFACT_NAME}${suffix}" ]] ||
+ fail "refusing to overwrite ${OUTPUT_DIR}/${ARTIFACT_NAME}${suffix}"
+done
+
+TEMP_DIR=$(mktemp -d)
+trap 'rm -rf "${TEMP_DIR}"' EXIT
+
+echo "Creating ${ARTIFACT_NAME} from ${GIT_REF}..."
+git -C "${SOURCE_ROOT}" -c tar.umask=0022 archive \
+ --format=tar \
+ --prefix="${ARCHIVE_ROOT}/" \
+ "${GIT_REF}" |
+ gzip -n >"${TEMP_DIR}/${ARTIFACT_NAME}"
+
+SHA512=$(calculate_sha512 "${TEMP_DIR}/${ARTIFACT_NAME}")
+printf '%s %s\n' "${SHA512}" "${ARTIFACT_NAME}" \
+ >"${TEMP_DIR}/${ARTIFACT_NAME}.sha512"
+
+if [[ -n "${SIGNING_KEY}" ]]; then
+ command -v gpg >/dev/null 2>&1 || fail "gpg is required to sign the artifact"
+ echo "Signing ${ARTIFACT_NAME} with ${SIGNING_KEY}..."
+ gpg --armor \
+ --local-user "${SIGNING_KEY}" \
+ --detach-sign \
+ --output "${TEMP_DIR}/${ARTIFACT_NAME}.asc" \
+ "${TEMP_DIR}/${ARTIFACT_NAME}"
+fi
+
+mv "${TEMP_DIR}/${ARTIFACT_NAME}" "${OUTPUT_DIR}/"
+mv "${TEMP_DIR}/${ARTIFACT_NAME}.sha512" "${OUTPUT_DIR}/"
+if [[ -n "${SIGNING_KEY}" ]]; then
+ mv "${TEMP_DIR}/${ARTIFACT_NAME}.asc" "${OUTPUT_DIR}/"
+fi
+
+echo "Created release artifacts in ${OUTPUT_DIR}:"
+echo " ${ARTIFACT_NAME}"
+echo " ${ARTIFACT_NAME}.sha512"
+if [[ -n "${SIGNING_KEY}" ]]; then
+ echo " ${ARTIFACT_NAME}.asc"
+else
+ echo " Signature not created; pass --signing-key for an official release candidate."
+fi
diff --git a/scripts/releasing/verify_release_candidate.sh b/scripts/releasing/verify_release_candidate.sh
new file mode 100755
index 0000000..cf6660d
--- /dev/null
+++ b/scripts/releasing/verify_release_candidate.sh
@@ -0,0 +1,227 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -euo pipefail
+
+ARTIFACT=""
+RAT_JAR=${RAT_JAR:-}
+ALLOW_UNSIGNED=false
+SKIP_RAT=false
+SKIP_BUILD=false
+
+usage() {
+ cat <<'EOF'
+Verify an Apache Paimon C++ source release candidate.
+
+Usage:
+ verify_release_candidate.sh [options] ARTIFACT
+
+Options:
+ --rat-jar FILE Apache RAT executable jar (or set RAT_JAR)
+ --allow-unsigned Allow a missing .asc file for local preparation only
+ --skip-rat Skip Apache RAT for local preparation only
+ --skip-build Skip the release build and test suite
+ -h, --help Show this help
+
+By default the script requires:
+ ARTIFACT.sha512
+ ARTIFACT.asc
+ an Apache RAT jar
+
+It verifies the checksum and signature, inspects and extracts the archive,
+checks release metadata, runs Apache RAT, then builds and tests from the
+extracted source distribution.
+EOF
+}
+
+fail() {
+ echo "Error: $*" >&2
+ exit 1
+}
+
+calculate_sha512() {
+ local file=$1
+ if command -v sha512sum >/dev/null 2>&1; then
+ sha512sum "${file}" | awk '{print $1}'
+ elif command -v shasum >/dev/null 2>&1; then
+ shasum -a 512 "${file}" | awk '{print $1}'
+ else
+ fail "sha512sum or shasum is required"
+ fi
+}
+
+while [[ $# -gt 0 ]]; do
+ case "$1" in
+ --rat-jar)
+ [[ $# -ge 2 ]] || fail "--rat-jar requires a value"
+ RAT_JAR=$2
+ shift 2
+ ;;
+ --allow-unsigned)
+ ALLOW_UNSIGNED=true
+ shift
+ ;;
+ --skip-rat)
+ SKIP_RAT=true
+ shift
+ ;;
+ --skip-build)
+ SKIP_BUILD=true
+ shift
+ ;;
+ -h|--help)
+ usage
+ exit 0
+ ;;
+ -*)
+ fail "unknown argument: $1"
+ ;;
+ *)
+ [[ -z "${ARTIFACT}" ]] || fail "only one artifact may be specified"
+ ARTIFACT=$1
+ shift
+ ;;
+ esac
+done
+
+[[ -n "${ARTIFACT}" ]] || fail "an artifact is required"
+[[ -f "${ARTIFACT}" ]] || fail "artifact does not exist: ${ARTIFACT}"
+
+ARTIFACT_DIR=$(cd "$(dirname "${ARTIFACT}")" && pwd)
+ARTIFACT_NAME=$(basename "${ARTIFACT}")
+ARTIFACT="${ARTIFACT_DIR}/${ARTIFACT_NAME}"
+
+if [[ "${ARTIFACT_NAME}" =~ ^apache-paimon-cpp-([0-9]+\.[0-9]+\.[0-9]+)-src\.tgz$ ]]; then
+ RELEASE_VERSION=${BASH_REMATCH[1]}
+else
+ fail "unexpected artifact name: ${ARTIFACT_NAME}"
+fi
+
+CHECKSUM_FILE="${ARTIFACT}.sha512"
+SIGNATURE_FILE="${ARTIFACT}.asc"
+ARCHIVE_ROOT="paimon-cpp-${RELEASE_VERSION}"
+
+[[ -f "${CHECKSUM_FILE}" ]] || fail "missing checksum: ${CHECKSUM_FILE}"
+EXPECTED_SHA512=$(awk 'NR == 1 {print $1}' "${CHECKSUM_FILE}")
+CHECKSUM_ARTIFACT=$(awk 'NR == 1 {print $2}' "${CHECKSUM_FILE}")
+[[ "${EXPECTED_SHA512}" =~ ^[0-9a-fA-F]{128}$ ]] ||
+ fail "invalid SHA-512 file: ${CHECKSUM_FILE}"
+[[ "${CHECKSUM_ARTIFACT}" == "${ARTIFACT_NAME}" ]] ||
+ fail "checksum file names ${CHECKSUM_ARTIFACT:-<missing>}, expected ${ARTIFACT_NAME}"
+ACTUAL_SHA512=$(calculate_sha512 "${ARTIFACT}")
+EXPECTED_SHA512=$(printf '%s' "${EXPECTED_SHA512}" | tr '[:upper:]' '[:lower:]')
+ACTUAL_SHA512=$(printf '%s' "${ACTUAL_SHA512}" | tr '[:upper:]' '[:lower:]')
+[[ "${ACTUAL_SHA512}" == "${EXPECTED_SHA512}" ]] ||
+ fail "SHA-512 checksum does not match"
+echo "SHA-512 checksum: valid"
+
+if [[ -f "${SIGNATURE_FILE}" ]]; then
+ command -v gpg >/dev/null 2>&1 || fail "gpg is required to verify the signature"
+ gpg --verify "${SIGNATURE_FILE}" "${ARTIFACT}"
+ echo "OpenPGP signature: valid"
+elif [[ "${ALLOW_UNSIGNED}" == true ]]; then
+ echo "OpenPGP signature: skipped for local preparation"
+else
+ fail "missing signature: ${SIGNATURE_FILE}"
+fi
+
+TEMP_DIR=$(mktemp -d)
+trap 'rm -rf "${TEMP_DIR}"' EXIT
+CONTENTS_FILE="${TEMP_DIR}/archive-contents.txt"
+
+tar -tzf "${ARTIFACT}" >"${CONTENTS_FILE}"
+[[ -s "${CONTENTS_FILE}" ]] || fail "source archive is empty"
+
+while IFS= read -r entry; do
+ [[ "${entry}" != /* ]] || fail "archive contains an absolute path: ${entry}"
+ [[ "${entry}" != ".." && "${entry}" != ../* && "${entry}" != */.. &&
+ "${entry}" != *"/../"* ]] ||
+ fail "archive contains path traversal: ${entry}"
+ [[ "${entry}" == "${ARCHIVE_ROOT}" || "${entry}" == "${ARCHIVE_ROOT}/"* ]] ||
+ fail "archive entry is outside ${ARCHIVE_ROOT}: ${entry}"
+done <"${CONTENTS_FILE}"
+
+tar -xzf "${ARTIFACT}" -C "${TEMP_DIR}"
+SOURCE_DIR="${TEMP_DIR}/${ARCHIVE_ROOT}"
+[[ -d "${SOURCE_DIR}" ]] || fail "archive root is missing: ${ARCHIVE_ROOT}"
+
+for required_file in LICENSE NOTICE CMakeLists.txt docs/source/conf.py; do
+ [[ -f "${SOURCE_DIR}/${required_file}" ]] ||
+ fail "required release file is missing: ${required_file}"
+done
+
+CMAKE_VERSION=$(
+ sed -n 's/^[[:space:]]*VERSION[[:space:]]\+\([0-9][0-9.]*\).*$/\1/p' \
+ "${SOURCE_DIR}/CMakeLists.txt" |
+ head -n 1
+)
+[[ "${CMAKE_VERSION}" == "${RELEASE_VERSION}" ]] ||
+ fail "CMake version ${CMAKE_VERSION:-<missing>} does not match ${RELEASE_VERSION}"
+
+DOCS_VERSION=$(
+ sed -n 's/^version = "\([^"]*\)"$/\1/p' "${SOURCE_DIR}/docs/source/conf.py" |
+ head -n 1
+)
+[[ "${DOCS_VERSION}" == "${RELEASE_VERSION}" ]] ||
+ fail "documentation version ${DOCS_VERSION:-<missing>} does not match ${RELEASE_VERSION}"
+
+UNEXPECTED_BINARIES=$(
+ find "${SOURCE_DIR}" -type f \
+ \( -name '*.a' -o -name '*.class' -o -name '*.dll' -o -name '*.dylib' \
+ -o -name '*.exe' -o -name '*.jar' -o -name '*.lib' -o -name '*.o' \
+ -o -name '*.pdb' -o -name '*.pyc' -o -name '*.so' \) \
+ -print
+)
+[[ -z "${UNEXPECTED_BINARIES}" ]] ||
+ fail "source archive contains unexpected compiled files:${UNEXPECTED_BINARIES}"
+
+UNSAFE_PERMISSIONS=$(
+ find "${SOURCE_DIR}" -type f \( -perm -0020 -o -perm -0002 \) -print
+)
+[[ -z "${UNSAFE_PERMISSIONS}" ]] ||
+ fail "source archive contains group- or world-writable files:${UNSAFE_PERMISSIONS}"
+
+echo "Archive layout and release metadata: valid"
+
+if [[ "${SKIP_RAT}" == false ]]; then
+ [[ -n "${RAT_JAR}" ]] || fail "--rat-jar or RAT_JAR is required"
+ [[ -f "${RAT_JAR}" ]] || fail "Apache RAT jar does not exist: ${RAT_JAR}"
+
+ RAT_REPORT="${TEMP_DIR}/rat-report.txt"
+ java -jar "${RAT_JAR}" \
+ -E "${SOURCE_DIR}/.github/.rat-excludes" \
+ -d "${SOURCE_DIR}" >"${RAT_REPORT}"
+
+ if grep -Eq 'Files with unapproved licenses:[[:space:]]*[1-9]' "${RAT_REPORT}"; then
+ cat "${RAT_REPORT}"
+ fail "Apache RAT found files with unapproved licenses"
+ fi
+ echo "Apache RAT: valid"
+else
+ echo "Apache RAT: skipped for local preparation"
+fi
+
+if [[ "${SKIP_BUILD}" == false ]]; then
+ "${SOURCE_DIR}/ci/scripts/build_paimon.sh" "${SOURCE_DIR}" false false Release
+ echo "Release build and tests: valid"
+else
+ echo "Release build and tests: skipped"
+fi
+
+echo "Release candidate verification completed successfully."
