ci/scripts/build_paimon.sh

#!/usr/bin/env bash
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

set -eux

usage() {
    echo "Usage: $0 --source_dir <path> [--enable_asan] [--enable_ubsan] [--enable_tsan] [--check_clang_tidy] [--build_type <type>] [--lint_git_target_commit <commit-or-branch>] [--install_smoke] [--verify_full_text_archives]"
}

source_dir=""
enable_asan="false"
enable_ubsan="false"
enable_tsan="false"
check_clang_tidy="false"
build_type="Debug"
lint_git_target_commit="origin/main"
install_smoke="false"
verify_full_text_archives="false"

while [[ $# -gt 0 ]]; do
    case "$1" in
        --source_dir)
            if [[ $# -lt 2 ]]; then
                echo "Missing value for --source_dir" >&2
                usage >&2
                exit 1
            fi
            source_dir=$2
            shift 2
            ;;
        --enable_asan)
            enable_asan="true"
            shift
            ;;
        --enable_ubsan)
            enable_ubsan="true"
            shift
            ;;
        --enable_tsan)
            enable_tsan="true"
            shift
            ;;
        --check_clang_tidy)
            check_clang_tidy="true"
            shift
            ;;
        --build_type)
            if [[ $# -lt 2 ]]; then
                echo "Missing value for --build_type" >&2
                usage >&2
                exit 1
            fi
            build_type=$2
            shift 2
            ;;
        --lint_git_target_commit)
            if [[ $# -lt 2 ]]; then
                echo "Missing value for --lint_git_target_commit" >&2
                usage >&2
                exit 1
            fi
            lint_git_target_commit=$2
            shift 2
            ;;
        --install_smoke)
            install_smoke="true"
            shift
            ;;
        --verify_full_text_archives)
            verify_full_text_archives="true"
            shift
            ;;
        -h | --help)
            usage
            exit 0
            ;;
        *)
            echo "Unknown argument: $1" >&2
            usage >&2
            exit 1
            ;;
    esac
done

if [[ -z "${source_dir}" ]]; then
    echo "--source_dir is required" >&2
    usage >&2
    exit 1
fi

if [[ "${enable_asan}" == "true" && "${enable_tsan}" == "true" ]]; then
    echo "ASAN and TSAN cannot be enabled together" >&2
    usage >&2
    exit 1
fi

build_dir="${source_dir}/build"

if [[ -n "${PAIMON_BUILD_JOBS:-}" ]]; then
    build_jobs="${PAIMON_BUILD_JOBS}"
elif command -v nproc >/dev/null 2>&1; then
    build_jobs=$(nproc)
elif command -v sysctl >/dev/null 2>&1; then
    build_jobs=$(sysctl -n hw.ncpu)
else
    build_jobs=4
fi

# Display ccache status if available
if command -v ccache &> /dev/null; then
    echo "=== ccache found: $(ccache --version | head -1) ==="
    ccache -p | grep -E "cache_dir|max_size|compression" || true
    ccache -z  # Reset statistics for this build
else
    echo "=== ccache not found, compiling without cache acceleration ==="
fi

mkdir -p "${build_dir}"
pushd "${build_dir}"

ENABLE_LUMINA="ON"
ENABLE_FULL_TEXT="ON"
ENABLE_LANCE="ON"
if [[ "${CC:-}" == *"gcc-8"* ]] || [[ "${CXX:-}" == *"g++-8"* ]]; then
    ENABLE_LUMINA="OFF"
    ENABLE_FULL_TEXT="OFF" # The full-text index (Rust FFI) is not built on the gcc-8 image.
fi
if [[ "${enable_tsan}" == "true" ]]; then
    ENABLE_FULL_TEXT="OFF" # The full-text Rust library is not TSAN-instrumented.
    ENABLE_LANCE="OFF"     # Lance's Rust library is not TSAN-instrumented.
fi
# CI always builds natively, so the host architecture is the target architecture.
host_arch=$(uname -m)
if [[ "${host_arch}" != "x86_64" ]]; then
    ENABLE_LUMINA="OFF"
    echo "=== Lumina disabled: no prebuilt artifacts for ${host_arch} ==="
fi
if [[ "${verify_full_text_archives}" == "true" && "${ENABLE_FULL_TEXT}" != "ON" ]]; then
    echo "--verify_full_text_archives requires the full-text index, which this build disables" >&2
    exit 1
fi

CMAKE_ARGS=(
    "-G Ninja"
    "-DCMAKE_BUILD_TYPE=${build_type}"
    "-DPAIMON_BUILD_TESTS=ON"
    "-DPAIMON_ENABLE_MOSAIC=ON"
    "-DPAIMON_ENABLE_LANCE=${ENABLE_LANCE}"
    "-DPAIMON_ENABLE_VORTEX=ON"
    "-DPAIMON_ENABLE_JINDO=ON"
    "-DPAIMON_ENABLE_OSS=ON"
    "-DPAIMON_ENABLE_S3=ON"
    "-DPAIMON_ENABLE_LUMINA=${ENABLE_LUMINA}"
    "-DPAIMON_ENABLE_LUCENE=ON"
    "-DPAIMON_ENABLE_FULL_TEXT=${ENABLE_FULL_TEXT}"
    "-DPAIMON_ENABLE_REST=ON"
    "-DPAIMON_LINT_GIT_TARGET_COMMIT=${lint_git_target_commit}"
)

if [[ "${enable_asan}" == "true" ]]; then
    CMAKE_ARGS+=("-DPAIMON_USE_ASAN=ON")
fi
if [[ "${enable_ubsan}" == "true" ]]; then
    CMAKE_ARGS+=("-DPAIMON_USE_UBSAN=ON")
fi
if [[ "${enable_tsan}" == "true" ]]; then
    CMAKE_ARGS+=("-DPAIMON_USE_TSAN=ON")
fi

cmake "${CMAKE_ARGS[@]}" "${source_dir}"
cmake --build . -- -j "${build_jobs}"
if [[ "${verify_full_text_archives}" == "true" ]]; then
    # TestCrossReadFixtures copies the full-text archives written by Paimon C++ here.
    full_text_archive_dir="${build_dir}/full_text_archives"
    mkdir -p "${full_text_archive_dir}"
    export PAIMON_FULL_TEXT_ARCHIVE_OUTPUT_DIR="${full_text_archive_dir}"
fi
ctest --output-on-failure -j "${build_jobs}"

if [[ "${verify_full_text_archives}" == "true" ]]; then
    # Read them with the Python binding of the same engine and compare the search results with
    # those of the checked-in fixtures.
    full_text_venv="${build_dir}/full_text_venv"
    python3 -m venv "${full_text_venv}"
    "${full_text_venv}/bin/pip" install paimon-ftindex==0.1.0
    "${full_text_venv}/bin/python" \
        "${source_dir}/test/test_data/full_text_fixtures/generate_fixtures.py" \
        --verify "${full_text_archive_dir}"
fi

if [[ "${check_clang_tidy}" == "true" ]]; then
    cmake --build . --target check-clang-tidy
fi

if [[ "${install_smoke}" == "true" ]]; then
    install_dir="${source_dir}/install-test"
    smoke_build_dir="${source_dir}/build-install-smoke"
    rm -rf "${install_dir}" "${smoke_build_dir}"

    cmake --install . --prefix "${install_dir}"
    cmake -G Ninja \
        -S "${source_dir}/scripts/releasing/install_smoke" \
        -B "${smoke_build_dir}" \
        -DCMAKE_BUILD_TYPE=Release \
        -DCMAKE_PREFIX_PATH="${install_dir};${build_dir}/arrow_ep-install"
    cmake --build "${smoke_build_dir}" -- -j "${build_jobs}"

    runtime_library_path="${install_dir}/lib:${install_dir}/lib64"
    runtime_library_path+=":${build_dir}/arrow_ep-install/lib"
    runtime_library_path+=":${build_dir}/arrow_ep-install/lib64"
    if [[ "$(uname -s)" == "Darwin" ]]; then
        cmake -E env \
            "DYLD_LIBRARY_PATH=${runtime_library_path}" \
            "${smoke_build_dir}/paimon_install_smoke"
    else
        cmake -E env \
            "LD_LIBRARY_PATH=${runtime_library_path}" \
            "${smoke_build_dir}/paimon_install_smoke"
    fi
fi

# Print ccache statistics after build
if command -v ccache &> /dev/null; then
    echo "=== ccache statistics after build ==="
    ccache -s
fi

popd

rm -rf "${build_dir}"
if [[ "${install_smoke}" == "true" ]]; then
    rm -rf "${install_dir}" "${smoke_build_dir}"
fi