test/inte/realtime_write_inte_test.cpp

/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */

#include <algorithm>
#include <atomic>
#include <chrono>
#include <condition_variable>
#include <cstdint>
#include <deque>
#include <functional>
#include <future>
#include <iterator>
#include <limits>
#include <map>
#include <memory>
#include <mutex>
#include <optional>
#include <set>
#include <string>
#include <thread>
#include <tuple>
#include <utility>
#include <vector>

#include "arrow/api.h"
#include "arrow/c/bridge.h"
#include "arrow/ipc/json_simple.h"
#include "fmt/format.h"
#include "gtest/gtest.h"
#include "paimon/catalog/catalog.h"
#include "paimon/catalog/identifier.h"
#include "paimon/commit_context.h"
#include "paimon/common/data/variant/variant_type_utils.h"
#include "paimon/common/table/special_fields.h"
#include "paimon/common/types/data_field.h"
#include "paimon/common/utils/path_util.h"
#include "paimon/common/utils/scope_guard.h"
#include "paimon/core/core_options.h"
#include "paimon/core/deletionvectors/bitmap_deletion_vector.h"
#include "paimon/core/deletionvectors/deletion_vectors_index_file.h"
#include "paimon/core/operation/commit/manifest_entry_changes.h"
#include "paimon/core/operation/commit/realtime_commit_properties.h"
#include "paimon/core/operation/commit/realtime_deduplicate_compaction_changes_provider.h"
#include "paimon/core/operation/metrics/commit_metrics.h"
#include "paimon/core/realtime/realtime_context_impl.h"
#include "paimon/core/realtime/realtime_schema_layout.h"
#include "paimon/core/schema/schema_manager.h"
#include "paimon/core/schema/table_schema.h"
#include "paimon/core/table/sink/commit_message_impl.h"
#include "paimon/core/table/source/data_split_impl.h"
#include "paimon/core/table/source/realtime_split.h"
#include "paimon/core/utils/snapshot_manager.h"
#include "paimon/data/shredding/map_shared_shredding_schema_utils.h"
#include "paimon/data/variant.h"
#include "paimon/defs.h"
#include "paimon/file_store_commit.h"
#include "paimon/file_store_write.h"
#include "paimon/fs/file_system.h"
#include "paimon/fs/local/local_file_system.h"
#include "paimon/memory/memory_pool.h"
#include "paimon/orphan_files_cleaner.h"
#include "paimon/predicate/function.h"
#include "paimon/predicate/predicate.h"
#include "paimon/predicate/predicate_builder.h"
#include "paimon/read_context.h"
#include "paimon/reader/count_reader.h"
#include "paimon/realtime/arrow_realtime_store_factory.h"
#include "paimon/realtime/realtime_context.h"
#include "paimon/realtime/realtime_snapshot_offsets.h"
#include "paimon/realtime/realtime_store.h"
#include "paimon/record_batch.h"
#include "paimon/scan_context.h"
#include "paimon/table/source/table_read.h"
#include "paimon/table/source/table_scan.h"
#include "paimon/testing/mock/mock_index_path_factory.h"
#include "paimon/testing/utils/blocking_file_system.h"
#include "paimon/testing/utils/dict_array_converter.h"
#include "paimon/testing/utils/read_result_collector.h"
#include "paimon/testing/utils/test_helper.h"
#include "paimon/testing/utils/testharness.h"
#include "paimon/testing/utils/variant_test_data.h"
#include "paimon/write_context.h"

namespace paimon::test {
namespace {

class TrackingRealtimeReadView final : public RealtimeReadView {
 public:
    explicit TrackingRealtimeReadView(std::shared_ptr<RealtimeReadView> delegate)
        : delegate_(std::move(delegate)) {}

    std::optional<OffsetRange> GetOffsetRange() const override {
        return delegate_->GetOffsetRange();
    }

    const std::shared_ptr<RealtimeReadView>& Delegate() const {
        return delegate_;
    }

 private:
    std::shared_ptr<RealtimeReadView> delegate_;
};

class DelegatingRealtimeStore : public RealtimeStore {
 public:
    explicit DelegatingRealtimeStore(const std::shared_ptr<RealtimeStore>& delegate)
        : delegate_(delegate) {}

    Status Write(RealtimeWriteBatch&& batch) override {
        return delegate_->Write(std::move(batch));
    }

    Result<std::optional<std::shared_ptr<RealtimeSegmentHandle>>> SealForCommit() override {
        return delegate_->SealForCommit();
    }

    Result<std::vector<std::unique_ptr<BatchReader>>> CreateCommitReaders(
        const std::shared_ptr<RealtimeSegmentHandle>& segment) override {
        return delegate_->CreateCommitReaders(segment);
    }

    Result<std::shared_ptr<RealtimeReadView>> AcquireReadView() override {
        return delegate_->AcquireReadView();
    }

    Result<std::vector<std::unique_ptr<BatchReader>>> CreateQueryReaders(
        const std::shared_ptr<RealtimeReadView>& view,
        const RealtimeQueryContext& context) override {
        return delegate_->CreateQueryReaders(view, context);
    }

    Status AdvanceCommittedOffset(int64_t committed_offset) override {
        return delegate_->AdvanceCommittedOffset(committed_offset);
    }

    RealtimeStoreDataUsage GetDataUsage() const override {
        return delegate_->GetDataUsage();
    }

    uint64_t GetMemoryUsage() const override {
        return delegate_->GetMemoryUsage();
    }

 protected:
    std::shared_ptr<RealtimeStore> delegate_;
};

class DecoratingRealtimeStoreFactory final : public RealtimeStoreFactory {
 public:
    using Decorator =
        std::function<std::shared_ptr<RealtimeStore>(const std::shared_ptr<RealtimeStore>&)>;

    explicit DecoratingRealtimeStoreFactory(Decorator decorator)
        : decorator_(std::move(decorator)) {}

    Result<std::shared_ptr<RealtimeStore>> Create(RealtimeStoreCreateRequest&& request) override {
        PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<RealtimeStore> delegate,
                               delegate_.Create(std::move(request)));
        return decorator_(delegate);
    }

 private:
    ArrowRealtimeStoreFactory delegate_;
    Decorator decorator_;
};

template <typename Store, typename... Args>
std::shared_ptr<RealtimeStoreFactory> MakeDecoratingFactory(Args... args) {
    return std::make_shared<DecoratingRealtimeStoreFactory>(
        [=](const std::shared_ptr<RealtimeStore>& delegate) -> std::shared_ptr<RealtimeStore> {
            return std::make_shared<Store>(delegate, args...);
        });
}

class QueryTrackingRealtimeStore final : public DelegatingRealtimeStore {
 public:
    QueryTrackingRealtimeStore(const std::shared_ptr<RealtimeStore>& delegate,
                               const std::shared_ptr<std::atomic<bool>>& saw_query_predicate,
                               const std::shared_ptr<std::weak_ptr<RealtimeReadView>>& query_view)
        : DelegatingRealtimeStore(delegate),
          saw_query_predicate_(saw_query_predicate),
          query_view_(query_view) {}

    Result<std::shared_ptr<RealtimeReadView>> AcquireReadView() override {
        PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<RealtimeReadView> delegate_view,
                               delegate_->AcquireReadView());
        return std::shared_ptr<RealtimeReadView>(
            std::make_shared<TrackingRealtimeReadView>(delegate_view));
    }

    Result<std::vector<std::unique_ptr<BatchReader>>> CreateQueryReaders(
        const std::shared_ptr<RealtimeReadView>& view,
        const RealtimeQueryContext& context) override {
        if (context.predicate) {
            saw_query_predicate_->store(true, std::memory_order_release);
        }
        *query_view_ = view;
        std::shared_ptr<TrackingRealtimeReadView> tracking_view =
            std::dynamic_pointer_cast<TrackingRealtimeReadView>(view);
        if (!tracking_view) {
            return Status::Invalid("query tracking store received an unexpected read view");
        }
        return delegate_->CreateQueryReaders(tracking_view->Delegate(), context);
    }

 private:
    std::shared_ptr<std::atomic<bool>> saw_query_predicate_;
    std::shared_ptr<std::weak_ptr<RealtimeReadView>> query_view_;
};

// Publishes a competing snapshot after the attempt has already materialized its DVs.
class InterleavingSnapshotFileSystem final : public LocalFileSystem {
 public:
    InterleavingSnapshotFileSystem(std::string snapshot_path, std::function<Status()> interleave)
        : snapshot_path_(std::move(snapshot_path)), interleave_(std::move(interleave)) {}

    Status AtomicStore(const std::string& path, const std::string& content) override {
        if (path == snapshot_path_ && interleave_) {
            std::function<Status()> interleave = std::move(interleave_);
            PAIMON_RETURN_NOT_OK(interleave());
            return Status::IOError("injected snapshot publication conflict");
        }
        return FileSystem::AtomicStore(path, content);
    }

 private:
    std::string snapshot_path_;
    std::function<Status()> interleave_;
};

class SnapshotConflictFileSystem final : public LocalFileSystem {
 public:
    explicit SnapshotConflictFileSystem(std::string conflict_snapshot_path)
        : conflict_snapshot_path_(std::move(conflict_snapshot_path)) {}

    void SetPassthroughChecksForCurrentThread(int32_t checks) {
        std::lock_guard<std::mutex> lock(mutex_);
        passthrough_checks_[std::this_thread::get_id()] = checks;
    }

    Result<bool> Exists(const std::string& path) const override {
        if (path != conflict_snapshot_path_) {
            return LocalFileSystem::Exists(path);
        }

        {
            std::unique_lock<std::mutex> lock(mutex_);
            if (conflict_snapshot_store_finished_) {
                lock.unlock();
                return LocalFileSystem::Exists(path);
            }
            if (conflict_snapshot_leader_selected_ &&
                conflict_snapshot_leader_ == std::this_thread::get_id()) {
                // Local AtomicStore rechecks the destination through virtual Exists during rename.
                lock.unlock();
                return LocalFileSystem::Exists(path);
            }
            auto passthrough_iter = passthrough_checks_.find(std::this_thread::get_id());
            if (passthrough_iter != passthrough_checks_.end() && passthrough_iter->second > 0) {
                --passthrough_iter->second;
                lock.unlock();
                return LocalFileSystem::Exists(path);
            }
            // Make both committers race for the same target snapshot. One observes it as absent
            // and commits it; the other waits for that store and then reports the conflict.
            ++conflict_snapshot_attempts_;
            cv_.notify_all();
            if (!cv_.wait_for(lock, std::chrono::seconds(10),
                              [this]() { return conflict_snapshot_attempts_ >= 2; })) {
                return Status::IOError("timed out waiting for concurrent snapshot commit");
            }
            if (!conflict_snapshot_leader_selected_) {
                conflict_snapshot_leader_selected_ = true;
                conflict_snapshot_leader_ = std::this_thread::get_id();
                return false;
            }
            if (!cv_.wait_for(lock, std::chrono::seconds(10),
                              [this]() { return conflict_snapshot_store_finished_; })) {
                return Status::IOError("timed out serializing conflicting snapshot commits");
            }
        }
        return LocalFileSystem::Exists(path);
    }

    Status AtomicStore(const std::string& path, const std::string& content) override {
        Status status = FileSystem::AtomicStore(path, content);
        if (path == conflict_snapshot_path_) {
            {
                std::lock_guard<std::mutex> lock(mutex_);
                conflict_snapshot_store_finished_ = true;
            }
            cv_.notify_all();
        }
        return status;
    }

    int32_t ConflictSnapshotAttempts() const {
        std::lock_guard<std::mutex> lock(mutex_);
        return conflict_snapshot_attempts_;
    }

 private:
    std::string conflict_snapshot_path_;
    mutable std::mutex mutex_;
    mutable std::condition_variable cv_;
    mutable std::map<std::thread::id, int32_t> passthrough_checks_;
    mutable int32_t conflict_snapshot_attempts_ = 0;
    mutable bool conflict_snapshot_leader_selected_ = false;
    mutable std::thread::id conflict_snapshot_leader_;
    mutable bool conflict_snapshot_store_finished_ = false;
};

}  // namespace

namespace {

constexpr char kDropPartitionCommitUser[] = "drop_partition_commit_user";
constexpr char kRollbackCommitUser[] = "rollback_commit_user";
constexpr char kTruncateCommitUser[] = "truncate_commit_user";

}  // namespace

class UnsupportedFunction : public Function {
 public:
    Type GetType() const override {
        return Type::EQUAL;
    }

    std::string ToString() const override {
        return "unsupported";
    }
};

class UnsupportedPredicate : public Predicate {
 public:
    bool operator==(const Predicate&) const override {
        return false;
    }

    const Function& GetFunction() const override {
        return function_;
    }

    std::shared_ptr<Predicate> Negate() const override {
        return nullptr;
    }

    std::string ToString() const override {
        return function_.ToString();
    }

 private:
    UnsupportedFunction function_;
};

class ConcurrentTestState {
 public:
    void WaitForStart() {
        ready_threads_.fetch_add(1, std::memory_order_release);
        while (!start_.load(std::memory_order_acquire)) {
            std::this_thread::yield();
        }
    }

    void StartWhenReady(int32_t worker_count) {
        while (ready_threads_.load(std::memory_order_acquire) < worker_count) {
            std::this_thread::yield();
        }
        start_.store(true, std::memory_order_release);
    }

    void RecordError(const Status& status) {
        RecordError(status.ToString());
    }

    void RecordError(std::string error) {
        {
            std::lock_guard<std::mutex> lock(mutex);
            errors_.push_back(std::move(error));
        }
        stop_.store(true, std::memory_order_release);
        progress_cv.notify_all();
        snapshot_cv.notify_all();
    }

    bool RecordErrorIfNotOk(const Status& status) {
        if (status.ok()) {
            return false;
        }
        RecordError(status);
        return true;
    }

    template <typename T>
    bool RecordErrorIfNotOk(const Result<T>& result) {
        if (result.ok()) {
            return false;
        }
        RecordError(result.status());
        return true;
    }

    bool ShouldStop() const {
        return stop_.load(std::memory_order_acquire);
    }

    const std::vector<std::string>& Errors() const {
        return errors_;
    }

    std::mutex mutex;
    std::condition_variable progress_cv;
    std::condition_variable snapshot_cv;

 private:
    std::atomic<bool> start_{false};
    std::atomic<bool> stop_{false};
    std::atomic<int32_t> ready_threads_{0};
    std::vector<std::string> errors_;
};

class RealtimeWriteInteTest : public ::testing::Test {
 protected:
    using Row = std::tuple<int64_t, std::string, std::string>;

    struct RealtimeMetricValues {
        double building_memory = 0;
        double sealed_memory = 0;
        double total_memory = 0;
        double building_rows = 0;
        double sealed_rows = 0;
        double total_rows = 0;
    };

    struct DiskFileLayout {
        bool has_level0_file = false;
        bool has_high_level_file = false;
        bool has_high_level_deletion_vector = false;
    };

    static Result<DiskFileLayout> InspectDiskFiles(
        const std::vector<std::shared_ptr<Split>>& splits) {
        DiskFileLayout layout;
        for (const std::shared_ptr<Split>& split : splits) {
            std::shared_ptr<DataSplitImpl> data_split =
                std::dynamic_pointer_cast<DataSplitImpl>(split);
            if (!data_split) {
                return Status::Invalid("expected a data split");
            }
            const std::vector<std::shared_ptr<DataFileMeta>>& files = data_split->DataFiles();
            const std::vector<std::optional<DeletionFile>>& deletion_files =
                data_split->DeletionFiles();
            if (!deletion_files.empty() && deletion_files.size() != files.size()) {
                return Status::Invalid("deletion files must be empty or match data files");
            }
            for (size_t i = 0; i < files.size(); ++i) {
                if (files[i]->level == 0) {
                    layout.has_level0_file = true;
                } else if (files[i]->level > 0) {
                    layout.has_high_level_file = true;
                    if (!deletion_files.empty() && deletion_files[i]) {
                        layout.has_high_level_deletion_vector = true;
                    }
                } else {
                    return Status::Invalid("data file has a negative level");
                }
            }
        }
        return layout;
    }

    void SetUp() override {
        pool_ = GetDefaultPool();
        dir_ = UniqueTestDirectory::Create("local");
        ASSERT_NE(nullptr, dir_);
        table_path_ = PathUtil::JoinPath(dir_->Str(), "foo.db/bar");
        fields_ = {arrow::field("id", arrow::int64()), arrow::field("payload", arrow::utf8()),
                   arrow::field("pt", arrow::utf8())};
        schema_ = arrow::schema(fields_);
        options_ = {
            {Options::FILE_FORMAT, "orc"},
            {Options::FILE_SYSTEM, "local"},
            {Options::BUCKET, "1"},
            {Options::BUCKET_KEY, "id"},
            {Options::TARGET_FILE_SIZE, "1048576"},
            {Options::REALTIME_ENABLED, "true"},
        };
    }

    void TearDown() override {
        dir_.reset();
    }

    void CreateTable(const std::vector<std::string>& partition_keys) const {
        auto c_schema = std::make_unique<ArrowSchema>();
        ASSERT_TRUE(arrow::ExportSchema(*schema_, c_schema.get()).ok());
        ASSERT_OK_AND_ASSIGN(std::unique_ptr<Catalog> catalog,
                             Catalog::Create(dir_->Str(), options_));
        ASSERT_OK(catalog->CreateDatabase("foo", {}, /*ignore_if_exists=*/false));
        ASSERT_OK(catalog->CreateTable(Identifier("foo", "bar"), c_schema.get(), partition_keys,
                                       /*primary_keys=*/{}, options_,
                                       /*ignore_if_exists=*/false));
    }

    void CreatePkTable(const std::vector<std::string>& partition_keys = {},
                       const std::vector<std::string>& primary_keys = {"id"}) const {
        auto c_schema = std::make_unique<ArrowSchema>();
        ASSERT_TRUE(arrow::ExportSchema(*schema_, c_schema.get()).ok());
        ASSERT_OK_AND_ASSIGN(std::unique_ptr<Catalog> catalog,
                             Catalog::Create(dir_->Str(), options_));
        ASSERT_OK(catalog->CreateDatabase("foo", {}, /*ignore_if_exists=*/false));
        std::vector<std::string> table_primary_keys = partition_keys;
        table_primary_keys.insert(table_primary_keys.end(), primary_keys.begin(),
                                  primary_keys.end());
        ASSERT_OK(catalog->CreateTable(Identifier("foo", "bar"), c_schema.get(), partition_keys,
                                       table_primary_keys, options_, /*ignore_if_exists=*/false));
    }

    void ConfigureRealtimeDeduplicateTable(const std::vector<std::string>& partition_keys = {}) {
        ConfigureRealtimeDeduplicateTable(partition_keys, "bitmap");
    }

    void ConfigureRealtimeDeduplicateTable(const std::vector<std::string>& partition_keys,
                                           const std::string& offset_index_type) {
        fields_ = {
            arrow::field(SpecialFields::RealtimeOffset().Name(), arrow::int64(),
                         /*nullable=*/false),
            arrow::field("id", arrow::int64(), /*nullable=*/false),
            arrow::field("payload", arrow::utf8()),
            arrow::field("pt", arrow::utf8()),
        };
        schema_ = arrow::schema(fields_);
        options_[Options::FILE_FORMAT] = "parquet";
        options_[Options::DELETION_VECTORS_ENABLED] = "true";
        options_[Options::REALTIME_DEDUPLICATE_KEY_FIELDS] = "id";
        options_["file-index." + offset_index_type + ".columns"] =
            SpecialFields::RealtimeOffset().Name();
        CreateTable(partition_keys);
    }

    Result<std::unique_ptr<FileStoreWrite>> CreateRealtimeWriter(
        const std::shared_ptr<RealtimeContext>& realtime_context) const {
        return CreateRealtimeWriter(realtime_context, /*temp_directory=*/"");
    }

    Result<std::unique_ptr<FileStoreWrite>> CreateRealtimeWriter(
        const std::shared_ptr<RealtimeContext>& realtime_context,
        const std::string& temp_directory) const {
        WriteContextBuilder builder(table_path_, commit_user_);
        builder.SetOptions(options_).WithStreamingMode(true).WithRealtimeContext(realtime_context);
        if (!temp_directory.empty()) {
            builder.WithTempDirectory(temp_directory);
        }
        PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<WriteContext> context, builder.Finish());
        return FileStoreWrite::Create(std::move(context));
    }

    Result<std::unique_ptr<FileStoreWrite>> CreateRealtimeWriter() const {
        PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<RealtimeContext> realtime_context,
                               RealtimeContext::Create());
        return CreateRealtimeWriter(realtime_context);
    }

    bool WaitForChannelFileCount(const std::string& temp_directory, int64_t expected_count) const {
        const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(2);
        while (std::chrono::steady_clock::now() < deadline) {
            if (TestHelper::CountChannelFiles(dir_->GetFileSystem(), temp_directory) ==
                expected_count) {
                return true;
            }
            std::this_thread::sleep_for(std::chrono::milliseconds(10));
        }
        return TestHelper::CountChannelFiles(dir_->GetFileSystem(), temp_directory) ==
               expected_count;
    }

    void ResetExternalOffset(const std::map<std::string, std::string>& partition, int32_t bucket,
                             int64_t next_offset) const {
        std::lock_guard<std::mutex> lock(external_offsets_mutex_);
        next_external_offsets_[RealtimePartitionBucket(partition, bucket)] = next_offset;
    }

    Result<std::unique_ptr<RecordBatch>> MakeBatch(const std::vector<Row>& rows,
                                                   bool partitioned) const {
        return MakeBatch(rows, partitioned, /*bucket=*/0);
    }

    Result<std::unique_ptr<RecordBatch>> MakeBatch(const std::vector<Row>& rows, bool partitioned,
                                                   int32_t bucket) const {
        return MakeBatch(rows, partitioned, bucket, /*row_kinds=*/{});
    }

    Result<std::unique_ptr<RecordBatch>> MakeBatch(
        const std::vector<Row>& rows, bool partitioned, int32_t bucket,
        const std::vector<RecordBatch::RowKind>& row_kinds) const {
        if (rows.empty()) {
            return Status::Invalid("cannot create an empty test batch");
        }
        const std::string& partition = std::get<2>(rows.front());
        std::map<std::string, std::string> batch_partition;
        if (partitioned) {
            batch_partition = {{"pt", partition}};
        }
        int64_t first_offset = 0;
        {
            std::lock_guard<std::mutex> lock(external_offsets_mutex_);
            int64_t& next_offset =
                next_external_offsets_[RealtimePartitionBucket(batch_partition, bucket)];
            first_offset = next_offset;
            next_offset += static_cast<int64_t>(rows.size());
        }
        std::string json = "[";
        for (size_t i = 0; i < rows.size(); ++i) {
            const auto& [id, payload, pt] = rows[i];
            if (partitioned && pt != partition) {
                return Status::Invalid("one test batch must contain only one partition");
            }
            if (i > 0) {
                json += ",";
            }
            json += fmt::format(R"([{}, {}, "{}", "{}"])", first_offset + static_cast<int64_t>(i),
                                id, payload, pt);
        }
        json += "]";

        const RealtimeStoreMode store_mode =
            options_.count(Options::REALTIME_DEDUPLICATE_KEY_FIELDS) > 0
                ? RealtimeStoreMode::DEDUPLICATE
                : RealtimeStoreMode::APPEND_ONLY;
        PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<RealtimeSchemaLayout> schema_layout,
                               RealtimeSchemaLayout::Create(store_mode, schema_));
        const std::shared_ptr<arrow::Schema>& realtime_schema = schema_layout->InputSchema();
        PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr<arrow::Array> array,
                                          arrow::ipc::internal::json::ArrayFromJSON(
                                              arrow::struct_(realtime_schema->fields()), json));
        ArrowArray c_array;
        PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*array, &c_array));
        return RecordBatchBuilder(&c_array)
            .SetRowKinds(row_kinds)
            .SetPartition(batch_partition)
            .SetBucket(bucket)
            .Finish();
    }

    Result<std::unique_ptr<RecordBatch>> MakeLogicalBatch(const std::vector<Row>& rows,
                                                          int32_t bucket) const {
        if (rows.empty()) {
            return Status::Invalid("cannot create an empty test batch");
        }
        std::string json = "[";
        for (size_t i = 0; i < rows.size(); ++i) {
            const auto& [id, payload, partition] = rows[i];
            if (i > 0) {
                json += ",";
            }
            json += fmt::format(R"([{}, "{}", "{}"])", id, payload, partition);
        }
        json += "]";
        PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(
            std::shared_ptr<arrow::Array> array,
            arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields_), json));
        ArrowArray c_array;
        PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*array, &c_array));
        return RecordBatchBuilder(&c_array).SetBucket(bucket).Finish();
    }

    Result<std::unique_ptr<RecordBatch>> MakeDatePartitionBatch(
        int64_t first_id, int64_t count, int32_t date, const std::string& partition) const {
        if (count <= 0) {
            return Status::Invalid("cannot create an empty test batch");
        }
        int64_t first_offset = 0;
        {
            std::lock_guard<std::mutex> lock(external_offsets_mutex_);
            int64_t& next_offset =
                next_external_offsets_[RealtimePartitionBucket({{"pt", partition}}, /*bucket=*/0)];
            first_offset = next_offset;
            next_offset += count;
        }
        std::string json = "[";
        for (int64_t i = 0; i < count; ++i) {
            if (i > 0) {
                json += ",";
            }
            int64_t id = first_id + i;
            json += fmt::format(R"([{}, {}, "value-{}", {}])", first_offset + i, id, id, date);
        }
        json += "]";

        PAIMON_ASSIGN_OR_RAISE(
            std::unique_ptr<RealtimeSchemaLayout> schema_layout,
            RealtimeSchemaLayout::Create(RealtimeStoreMode::APPEND_ONLY, schema_));
        const std::shared_ptr<arrow::Schema>& realtime_schema = schema_layout->InputSchema();
        PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr<arrow::Array> array,
                                          arrow::ipc::internal::json::ArrayFromJSON(
                                              arrow::struct_(realtime_schema->fields()), json));
        ArrowArray c_array;
        PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*array, &c_array));
        return RecordBatchBuilder(&c_array).SetPartition({{"pt", partition}}).SetBucket(0).Finish();
    }

    Result<std::unique_ptr<RecordBatch>> MakeUnpartitionedBatchFromJson(
        const std::string& json) const {
        PAIMON_ASSIGN_OR_RAISE(
            std::unique_ptr<RealtimeSchemaLayout> schema_layout,
            RealtimeSchemaLayout::Create(RealtimeStoreMode::APPEND_ONLY, schema_));
        const std::shared_ptr<arrow::Schema>& realtime_schema = schema_layout->InputSchema();
        PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr<arrow::Array> array,
                                          arrow::ipc::internal::json::ArrayFromJSON(
                                              arrow::struct_(realtime_schema->fields()), json));
        ArrowArray c_array;
        PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*array, &c_array));
        return RecordBatchBuilder(&c_array).SetBucket(0).Finish();
    }

    static std::vector<Row> MakeRows(int64_t first_id, int64_t count,
                                     const std::string& partition) {
        std::vector<Row> rows;
        rows.reserve(count);
        for (int64_t i = 0; i < count; ++i) {
            int64_t id = first_id + i;
            rows.emplace_back(id, "value-" + std::to_string(id), partition);
        }
        return rows;
    }

    Result<int64_t> Commit(const std::vector<RealtimeCommitProgress>& realtime_commits,
                           int64_t commit_identifier) const {
        CommitContextBuilder builder(table_path_, commit_user_);
        PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<CommitContext> context,
                               builder.SetOptions(options_).Finish());
        PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<FileStoreCommit> commit,
                               FileStoreCommit::Create(std::move(context)));
        return commit->CommitWithProgress(realtime_commits, commit_identifier,
                                          /*watermark=*/std::nullopt);
    }

    Result<int64_t> WriteCommitAndRefresh(const std::vector<Row>& rows, bool partitioned,
                                          int32_t bucket,
                                          const std::vector<RecordBatch::RowKind>& row_kinds,
                                          int64_t commit_identifier, FileStoreWrite* writer) const {
        PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<RecordBatch> batch,
                               MakeBatch(rows, partitioned, bucket, row_kinds));
        PAIMON_RETURN_NOT_OK(writer->Write(std::move(batch)));
        PAIMON_ASSIGN_OR_RAISE(std::vector<RealtimeCommitProgress> progress,
                               writer->PrepareCommitWithProgress(commit_identifier));
        PAIMON_ASSIGN_OR_RAISE(int64_t snapshot, Commit(progress, commit_identifier));
        PAIMON_RETURN_NOT_OK(writer->RefreshCommittedSnapshot(snapshot));
        return snapshot;
    }

    Result<int64_t> DropPartition(const std::map<std::string, std::string>& partition,
                                  int64_t commit_identifier) const {
        CommitContextBuilder builder(table_path_, kDropPartitionCommitUser);
        PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<CommitContext> context,
                               builder.SetOptions(options_).Finish());
        PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<FileStoreCommit> commit,
                               FileStoreCommit::Create(std::move(context)));
        PAIMON_RETURN_NOT_OK(commit->DropPartition({partition}, commit_identifier));
        PAIMON_ASSIGN_OR_RAISE(CoreOptions options, CoreOptions::FromMap(options_));
        SnapshotManager snapshot_manager(options.GetFileSystem(), table_path_);
        PAIMON_ASSIGN_OR_RAISE(std::optional<Snapshot> latest_snapshot,
                               snapshot_manager.LatestSnapshot());
        if (!latest_snapshot) {
            return Status::Invalid("drop partition did not produce a snapshot");
        }
        return latest_snapshot->Id();
    }

    Result<int64_t> TruncateTable(int64_t commit_identifier) const {
        CommitContextBuilder builder(table_path_, kTruncateCommitUser);
        PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<CommitContext> context,
                               builder.SetOptions(options_).Finish());
        PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<FileStoreCommit> commit,
                               FileStoreCommit::Create(std::move(context)));
        PAIMON_RETURN_NOT_OK(commit->TruncateTable(commit_identifier));
        PAIMON_ASSIGN_OR_RAISE(CoreOptions options, CoreOptions::FromMap(options_));
        SnapshotManager snapshot_manager(options.GetFileSystem(), table_path_);
        PAIMON_ASSIGN_OR_RAISE(std::optional<Snapshot> latest_snapshot,
                               snapshot_manager.LatestSnapshot());
        if (!latest_snapshot) {
            return Status::Invalid("truncate did not produce a snapshot");
        }
        return latest_snapshot->Id();
    }

    Result<int64_t> RollbackToAsLatest(int64_t target_snapshot_id) const {
        CommitContextBuilder builder(table_path_, kRollbackCommitUser);
        PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<CommitContext> context,
                               builder.SetOptions(options_).Finish());
        PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<FileStoreCommit> commit,
                               FileStoreCommit::Create(std::move(context)));
        PAIMON_ASSIGN_OR_RAISE(bool rolled_back, commit->RollbackToAsLatest(target_snapshot_id));
        if (!rolled_back) {
            return Status::Invalid("failed to commit rollback snapshot");
        }
        PAIMON_ASSIGN_OR_RAISE(CoreOptions options, CoreOptions::FromMap(options_));
        SnapshotManager snapshot_manager(options.GetFileSystem(), table_path_);
        PAIMON_ASSIGN_OR_RAISE(std::optional<Snapshot> latest_snapshot,
                               snapshot_manager.LatestSnapshot());
        if (!latest_snapshot) {
            return Status::Invalid("rollback did not produce a snapshot");
        }
        return latest_snapshot->Id();
    }

    Result<std::vector<std::shared_ptr<CommitMessage>>> CompactAndPrepareCommit(
        const std::vector<RealtimePartitionBucket>& partition_buckets, int64_t commit_identifier,
        bool full_compaction, std::unique_ptr<FileStoreWrite>* compactor) const {
        WriteContextBuilder builder(table_path_, "regression-compactor");
        PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<WriteContext> context,
                               builder.SetOptions(options_).WithStreamingMode(true).Finish());
        PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<FileStoreWrite> writer,
                               FileStoreWrite::Create(std::move(context)));
        for (const RealtimePartitionBucket& scope : partition_buckets) {
            PAIMON_RETURN_NOT_OK(writer->Compact(scope.partition, scope.bucket, full_compaction));
        }
        PAIMON_ASSIGN_OR_RAISE(std::vector<std::shared_ptr<CommitMessage>> messages,
                               writer->PrepareCommit(/*wait_compaction=*/true, commit_identifier));
        *compactor = std::move(writer);
        return messages;
    }

    Result<Snapshot> CommitCompaction(const std::vector<std::shared_ptr<CommitMessage>>& messages,
                                      int64_t commit_identifier) const {
        CommitContextBuilder builder(table_path_, "regression-compactor");
        PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<CommitContext> context,
                               builder.SetOptions(options_).Finish());
        PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<FileStoreCommit> commit,
                               FileStoreCommit::Create(std::move(context)));
        PAIMON_RETURN_NOT_OK(commit->Commit(messages, commit_identifier));
        SnapshotManager snapshot_manager(dir_->GetFileSystem(), table_path_);
        PAIMON_ASSIGN_OR_RAISE(std::optional<Snapshot> snapshot, snapshot_manager.LatestSnapshot());
        if (!snapshot) {
            return Status::Invalid("compaction did not produce a snapshot");
        }
        return *snapshot;
    }

    Result<Snapshot> CompactAndCommit(const std::map<std::string, std::string>& partition,
                                      int32_t bucket, int64_t commit_identifier,
                                      bool full_compaction = true) const {
        WriteContextBuilder write_builder(table_path_, commit_user_);
        write_builder.SetOptions(options_).WithStreamingMode(true).WithTempDirectory(
            PathUtil::JoinPath(dir_->Str(), "compact-tmp"));
        PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<WriteContext> write_context, write_builder.Finish());
        PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<FileStoreWrite> compaction_writer,
                               FileStoreWrite::Create(std::move(write_context)));
        PAIMON_RETURN_NOT_OK(compaction_writer->Compact(partition, bucket, full_compaction));
        PAIMON_ASSIGN_OR_RAISE(
            std::vector<std::shared_ptr<CommitMessage>> compaction_messages,
            compaction_writer->PrepareCommit(/*wait_compaction=*/true, commit_identifier));
        if (compaction_messages.empty()) {
            return Status::Invalid("compaction did not produce a commit message");
        }

        CommitContextBuilder commit_builder(table_path_, commit_user_);
        PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<CommitContext> commit_context,
                               commit_builder.SetOptions(options_).Finish());
        PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<FileStoreCommit> compaction_commit,
                               FileStoreCommit::Create(std::move(commit_context)));
        PAIMON_RETURN_NOT_OK(compaction_commit->Commit(compaction_messages, commit_identifier));
        PAIMON_RETURN_NOT_OK(compaction_writer->Close());

        PAIMON_ASSIGN_OR_RAISE(CoreOptions core_options, CoreOptions::FromMap(options_));
        SnapshotManager snapshot_manager(core_options.GetFileSystem(), table_path_);
        PAIMON_ASSIGN_OR_RAISE(std::optional<Snapshot> compact_snapshot,
                               snapshot_manager.LatestSnapshot());
        if (!compact_snapshot) {
            return Status::Invalid("compaction did not produce a snapshot");
        }
        return compact_snapshot.value();
    }

    Result<Snapshot> CompactCommitAndRefresh(const std::map<std::string, std::string>& partition,
                                             int32_t bucket, int64_t commit_identifier,
                                             bool full_compaction, FileStoreWrite* writer) const {
        PAIMON_ASSIGN_OR_RAISE(
            Snapshot snapshot,
            CompactAndCommit(partition, bucket, commit_identifier, full_compaction));
        PAIMON_RETURN_NOT_OK(writer->RefreshCommittedSnapshot(snapshot.Id()));
        return snapshot;
    }

    Result<int32_t> ExpireSnapshots() const {
        CommitContextBuilder builder(table_path_, commit_user_);
        PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<CommitContext> context,
                               builder.SetOptions(options_).Finish());
        PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<FileStoreCommit> commit,
                               FileStoreCommit::Create(std::move(context)));
        return commit->Expire();
    }

    Result<std::shared_ptr<Plan>> CreatePlan(
        const std::shared_ptr<RealtimeContext>& realtime_context,
        const std::shared_ptr<Predicate>& predicate) const {
        ScanContextBuilder scan_builder(table_path_);
        if (realtime_context) {
            scan_builder.WithRealtimeContext(realtime_context);
        }
        scan_builder.SetPredicate(predicate).WithMemoryPool(pool_);
        PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<ScanContext> scan_context,
                               scan_builder.SetOptions(options_).Finish());
        PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<TableScan> scan,
                               TableScan::Create(std::move(scan_context)));
        return scan->CreatePlan();
    }

    Result<std::unique_ptr<BatchReader>> CreateQueryReader(
        const std::shared_ptr<Plan>& plan,
        const std::shared_ptr<RealtimeContext>& realtime_context) const {
        ReadContextBuilder read_builder(table_path_);
        read_builder.SetOptions(options_)
            .SetReadFieldNames({"id", "payload", "pt"})
            .WithRealtimeContext(realtime_context)
            .WithMemoryPool(pool_);
        PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<ReadContext> read_context, read_builder.Finish());
        PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<TableRead> table_read,
                               TableRead::Create(std::move(read_context)));
        return table_read->CreateReader(plan->Splits());
    }

    Result<std::unique_ptr<BatchReader>> CreateQueryReader(
        const std::shared_ptr<RealtimeContext>& realtime_context) const {
        PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<Plan> plan,
                               CreatePlan(realtime_context, /*predicate=*/nullptr));
        return CreateQueryReader(plan, realtime_context);
    }

    Result<std::shared_ptr<arrow::ChunkedArray>> ReadPlan(
        const std::shared_ptr<Plan>& plan, const std::shared_ptr<RealtimeContext>& realtime_context,
        const std::vector<std::string>& read_fields, const std::shared_ptr<Predicate>& predicate,
        bool enable_predicate_filter) const {
        ReadContextBuilder read_builder(table_path_);
        read_builder.SetOptions(options_)
            .SetReadFieldNames(read_fields)
            .SetPredicate(predicate)
            .EnablePredicateFilter(enable_predicate_filter)
            .WithRealtimeContext(realtime_context)
            .WithMemoryPool(pool_);
        PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<ReadContext> read_context, read_builder.Finish());
        PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<TableRead> table_read,
                               TableRead::Create(std::move(read_context)));
        PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<BatchReader> reader,
                               table_read->CreateReader(plan->Splits()));
        PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<arrow::ChunkedArray> result,
                               ReadResultCollector::CollectResult(std::move(reader)));
        return result;
    }

    void ReadPlanWithSchemaAndCheck(const std::shared_ptr<Plan>& plan,
                                    const std::shared_ptr<RealtimeContext>& realtime_context,
                                    const std::shared_ptr<arrow::Schema>& read_schema,
                                    const std::string& expected_json) const {
        std::unique_ptr<ArrowSchema> c_read_schema = std::make_unique<ArrowSchema>();
        ASSERT_TRUE(arrow::ExportSchema(*read_schema, c_read_schema.get()).ok());
        ReadContextBuilder read_builder(table_path_);
        read_builder.SetOptions(options_)
            .SetReadSchema(std::move(c_read_schema))
            .WithRealtimeContext(realtime_context)
            .WithMemoryPool(pool_);
        ASSERT_OK_AND_ASSIGN(std::unique_ptr<ReadContext> read_context, read_builder.Finish());
        ASSERT_OK_AND_ASSIGN(std::unique_ptr<TableRead> table_read,
                             TableRead::Create(std::move(read_context)));
        ASSERT_OK_AND_ASSIGN(std::unique_ptr<BatchReader> reader,
                             table_read->CreateReader(plan->Splits()));
        ASSERT_OK_AND_ASSIGN(std::shared_ptr<arrow::ChunkedArray> result,
                             ReadResultCollector::CollectResult(std::move(reader)));

        arrow::FieldVector result_fields = {arrow::field("_VALUE_KIND", arrow::int8())};
        result_fields.insert(result_fields.end(), read_schema->fields().begin(),
                             read_schema->fields().end());
        std::shared_ptr<arrow::Array> expected =
            arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(result_fields), expected_json)
                .ValueOrDie();
        // Match the dictionary and nullability normalization applied by ReadResultCollector.
        ASSERT_OK_AND_ASSIGN(
            expected, DictArrayConverter::ConvertDictArray(expected, arrow::default_memory_pool()));
        ASSERT_NE(nullptr, result);
        ASSERT_TRUE(std::make_shared<arrow::ChunkedArray>(expected)->Equals(*result))
            << result->ToString();
    }

    Result<std::vector<Row>> ReadRows(
        const std::shared_ptr<RealtimeContext>& realtime_context) const {
        PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<Plan> plan,
                               CreatePlan(realtime_context, /*predicate=*/nullptr));
        return ReadRows(plan, realtime_context);
    }

    Result<std::vector<Row>> ReadRows(
        const std::shared_ptr<Plan>& plan,
        const std::shared_ptr<RealtimeContext>& realtime_context) const {
        return ReadRows(plan, realtime_context, /*predicate=*/nullptr,
                        /*enable_predicate_filter=*/false);
    }

    Result<std::vector<Row>> ReadRows(const std::shared_ptr<Plan>& plan,
                                      const std::shared_ptr<RealtimeContext>& realtime_context,
                                      const std::shared_ptr<Predicate>& predicate,
                                      bool enable_predicate_filter) const {
        PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<arrow::ChunkedArray> read_result,
                               ReadPlan(plan, realtime_context, {"id", "payload", "pt"}, predicate,
                                        enable_predicate_filter));
        const std::shared_ptr<arrow::ChunkedArray>& result = read_result;

        std::vector<Row> rows;
        if (!result) {
            return rows;
        }
        for (const std::shared_ptr<arrow::Array>& chunk : result->chunks()) {
            std::shared_ptr<arrow::StructArray> data =
                std::dynamic_pointer_cast<arrow::StructArray>(chunk);
            if (!data || data->num_fields() != 4) {
                return Status::Invalid("unexpected real-time test read schema");
            }
            std::shared_ptr<arrow::Int8Array> row_kinds =
                std::dynamic_pointer_cast<arrow::Int8Array>(data->field(0));
            std::shared_ptr<arrow::Int64Array> ids =
                std::dynamic_pointer_cast<arrow::Int64Array>(data->field(1));
            std::shared_ptr<arrow::StringArray> payloads =
                std::dynamic_pointer_cast<arrow::StringArray>(data->field(2));
            std::shared_ptr<arrow::StringArray> partitions =
                std::dynamic_pointer_cast<arrow::StringArray>(data->field(3));
            if (!row_kinds || !ids || !payloads || !partitions) {
                return Status::Invalid("unexpected real-time test read field type");
            }
            for (int64_t i = 0; i < data->length(); ++i) {
                if (row_kinds->IsNull(i) ||
                    row_kinds->Value(i) != static_cast<int8_t>(RecordBatch::RowKind::INSERT) ||
                    ids->IsNull(i) || payloads->IsNull(i) || partitions->IsNull(i)) {
                    return Status::Invalid("unexpected null or row kind in real-time test result");
                }
                rows.emplace_back(ids->Value(i), payloads->GetString(i), partitions->GetString(i));
            }
        }
        return rows;
    }

    Result<std::vector<Row>> ReadRows() const {
        return ReadRows(/*realtime_context=*/nullptr);
    }

    Result<int64_t> CountRows(const std::shared_ptr<Plan>& plan,
                              const std::shared_ptr<RealtimeContext>& realtime_context) const {
        ReadContextBuilder read_builder(table_path_);
        read_builder.SetOptions(options_)
            .WithRealtimeContext(realtime_context)
            .WithMemoryPool(pool_);
        PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<ReadContext> read_context, read_builder.Finish());
        PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<TableRead> table_read,
                               TableRead::Create(std::move(read_context)));
        PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<CountReader> count_reader,
                               table_read->CreateCountReader(plan->Splits()));
        return count_reader->CountRows();
    }

    Result<uint64_t> GetRealtimeMemoryUsage(
        const std::shared_ptr<RealtimeContext>& realtime_context) const {
        PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<RealtimeContextImpl> realtime_context_impl,
                               RealtimeContextImpl::Cast(realtime_context));
        PAIMON_ASSIGN_OR_RAISE(RealtimeReadState read_state,
                               realtime_context_impl->AcquireReadState());
        uint64_t memory_usage = 0;
        for (const RealtimePartitionBucketView& view : read_state.views) {
            memory_usage += view.store->GetMemoryUsage();
        }
        return memory_usage;
    }

    static Result<RealtimeMetricValues> ReadRealtimeMetrics(
        const std::shared_ptr<Metrics>& metrics) {
        if (!metrics) {
            return Status::Invalid("real-time metrics are null");
        }
        RealtimeMetricValues result;
        PAIMON_ASSIGN_OR_RAISE(result.building_memory,
                               metrics->GetGauge(RealtimeMetrics::kBuildingMemoryBytes));
        PAIMON_ASSIGN_OR_RAISE(result.sealed_memory,
                               metrics->GetGauge(RealtimeMetrics::kSealedMemoryBytes));
        PAIMON_ASSIGN_OR_RAISE(result.total_memory,
                               metrics->GetGauge(RealtimeMetrics::kTotalMemoryBytes));
        PAIMON_ASSIGN_OR_RAISE(result.building_rows,
                               metrics->GetGauge(RealtimeMetrics::kBuildingRowCount));
        PAIMON_ASSIGN_OR_RAISE(result.sealed_rows,
                               metrics->GetGauge(RealtimeMetrics::kSealedRowCount));
        PAIMON_ASSIGN_OR_RAISE(result.total_rows,
                               metrics->GetGauge(RealtimeMetrics::kTotalRowCount));
        return result;
    }

    Result<std::vector<int64_t>> ReadPkSequences(
        const std::shared_ptr<RealtimeContext>& realtime_context) const {
        PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<RealtimeContextImpl> realtime_context_impl,
                               RealtimeContextImpl::Cast(realtime_context));
        PAIMON_ASSIGN_OR_RAISE(RealtimeReadState read_state,
                               realtime_context_impl->AcquireReadState());
        const std::vector<RealtimePartitionBucketView>& views = read_state.views;
        if (views.size() != 1) {
            return Status::Invalid("expected one PK real-time read view");
        }
        PAIMON_ASSIGN_OR_RAISE(CoreOptions core_options, CoreOptions::FromMap(options_));
        SchemaManager schema_manager(core_options.GetFileSystem(), table_path_);
        PAIMON_ASSIGN_OR_RAISE(std::optional<std::shared_ptr<TableSchema>> table_schema,
                               schema_manager.Latest());
        if (!table_schema) {
            return Status::Invalid("expected a table schema");
        }
        auto read_schema = std::make_unique<ArrowSchema>();
        std::shared_ptr<arrow::Schema> value_schema =
            DataField::ConvertDataFieldsToArrowSchema(table_schema.value()->Fields());
        PAIMON_ASSIGN_OR_RAISE(
            std::unique_ptr<RealtimeSchemaLayout> schema_layout,
            RealtimeSchemaLayout::Create(RealtimeStoreMode::PRIMARY_KEY, value_schema));
        PAIMON_RETURN_NOT_OK_FROM_ARROW(
            arrow::ExportSchema(*schema_layout->StoreWriteSchema(), read_schema.get()));
        ScopeGuard schema_guard([schema = read_schema.get()]() { ArrowSchemaRelease(schema); });
        RealtimeQueryContext query_context{read_schema.get(), /*predicate=*/nullptr};
        PAIMON_ASSIGN_OR_RAISE(
            std::vector<std::unique_ptr<BatchReader>> readers,
            views[0].store->CreateQueryReaders(views[0].read_view, query_context));
        std::vector<int64_t> sequences;
        for (const std::unique_ptr<BatchReader>& reader : readers) {
            while (true) {
                PAIMON_ASSIGN_OR_RAISE(BatchReader::ReadBatch batch, reader->NextBatch());
                if (BatchReader::IsEofBatch(batch)) {
                    break;
                }
                PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(
                    std::shared_ptr<arrow::Array> imported,
                    arrow::ImportArray(batch.first.get(), batch.second.get()));
                std::shared_ptr<arrow::StructArray> values =
                    std::dynamic_pointer_cast<arrow::StructArray>(imported);
                if (!values) {
                    return Status::Invalid("PK query reader did not return a StructArray");
                }
                std::shared_ptr<arrow::Int64Array> sequence_array =
                    std::dynamic_pointer_cast<arrow::Int64Array>(
                        values->GetFieldByName(SpecialFields::SequenceNumber().Name()));
                if (!sequence_array) {
                    return Status::Invalid("PK query reader did not return sequence numbers");
                }
                for (int64_t row = 0; row < sequence_array->length(); ++row) {
                    sequences.push_back(sequence_array->Value(row));
                }
            }
            reader->Close();
        }
        return sequences;
    }

    static std::vector<std::shared_ptr<DataFileMeta>> NewFiles(
        const std::vector<RealtimeCommitProgress>& progresses) {
        std::vector<std::shared_ptr<DataFileMeta>> files;
        for (const RealtimeCommitProgress& progress : progresses) {
            std::shared_ptr<CommitMessageImpl> message =
                std::dynamic_pointer_cast<CommitMessageImpl>(progress.commit_message);
            if (!message) {
                continue;
            }
            const std::vector<std::shared_ptr<DataFileMeta>>& new_files =
                message->GetNewFilesIncrement().NewFiles();
            files.insert(files.end(), new_files.begin(), new_files.end());
        }
        return files;
    }

    static Status ValidateReadPrefix(const std::vector<Row>& rows, int64_t total_rows) {
        std::vector<bool> seen(static_cast<size_t>(total_rows), false);
        int64_t max_id = -1;
        for (const Row& row : rows) {
            const auto& [id, payload, partition] = row;
            if (id < 0 || id >= total_rows) {
                return Status::Invalid("real-time read id is out of range");
            }
            if (seen[static_cast<size_t>(id)]) {
                return Status::Invalid("real-time read contains duplicate ids");
            }
            if (payload != "value-" + std::to_string(id) || partition != "p0") {
                return Status::Invalid("real-time read row does not match its id");
            }
            seen[static_cast<size_t>(id)] = true;
            max_id = std::max(max_id, id);
        }
        for (int64_t id = 0; id <= max_id; ++id) {
            if (!seen[static_cast<size_t>(id)]) {
                return Status::Invalid("real-time read contains an id gap");
            }
        }
        return Status::OK();
    }

    struct FileState {
        std::shared_ptr<DataFileMeta> file;
        std::string bucket_path;
        std::optional<DeletionFile> deletion_file;
        std::set<int64_t> deleted_positions;
    };

    Result<std::vector<FileState>> InspectFiles() const {
        PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<Plan> plan,
                               CreatePlan(/*realtime_context=*/nullptr, /*predicate=*/nullptr));
        std::vector<FileState> states;
        for (const auto& split : plan->Splits()) {
            auto data = std::dynamic_pointer_cast<DataSplitImpl>(split);
            if (!data || (!data->DeletionFiles().empty() &&
                          data->DeletionFiles().size() != data->DataFiles().size())) {
                return Status::Invalid("expected data files with matching deletion files");
            }
            for (size_t i = 0; i < data->DataFiles().size(); ++i) {
                FileState state{data->DataFiles()[i], data->BucketPath(), std::nullopt, {}};
                if (!data->DeletionFiles().empty() && data->DeletionFiles()[i]) {
                    state.deletion_file = data->DeletionFiles()[i];
                    PAIMON_ASSIGN_OR_RAISE(PAIMON_UNIQUE_PTR<DeletionVector> dv,
                                           DeletionVector::Read(dir_->GetFileSystem().get(),
                                                                *state.deletion_file, pool_.get()));
                    PAIMON_RETURN_NOT_OK(dv->ForEachDeletedPosition(
                        [&](int64_t position) { state.deleted_positions.insert(position); }));
                }
                states.push_back(std::move(state));
            }
        }
        return states;
    }

    Result<RealtimeOffsetMap> ReadCommittedOffsets() const {
        PAIMON_ASSIGN_OR_RAISE(CoreOptions options, CoreOptions::FromMap(options_));
        SnapshotManager snapshot_manager(options.GetFileSystem(), table_path_);
        PAIMON_ASSIGN_OR_RAISE(std::optional<Snapshot> snapshot, snapshot_manager.LatestSnapshot());
        return RealtimeCommitProperties::ReadOffsets(snapshot, options.GetFileSystem());
    }

    Status PrepareAndClose(FileStoreWrite* writer) const {
        PAIMON_ASSIGN_OR_RAISE(std::vector<RealtimeCommitProgress> progress,
                               writer->PrepareCommitWithProgress(/*commit_identifier=*/0));
        static_cast<void>(progress);
        return writer->Close();
    }

    void FinalizeCommitAndCheck(FileStoreWrite* writer,
                                std::vector<RealtimeCommitProgress> realtime_commits,
                                int64_t prepare_identifier, std::vector<Row> expected_rows) const {
        ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> final_commits,
                             writer->PrepareCommitWithProgress(prepare_identifier));
        realtime_commits.insert(realtime_commits.end(),
                                std::make_move_iterator(final_commits.begin()),
                                std::make_move_iterator(final_commits.end()));
        ASSERT_OK(Commit(realtime_commits, prepare_identifier));
        ASSERT_OK(writer->Close());

        ASSERT_OK_AND_ASSIGN(std::vector<Row> actual_rows, ReadRows());
        ASSERT_EQ(expected_rows, actual_rows);
    }

    void ReplayPkWalAndCommit(const std::vector<Row>& wal,
                              const std::vector<RecordBatch::RowKind>& row_kinds,
                              int64_t commit_identifier,
                              const std::vector<Row>& expected_rows) const {
        ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
                             RealtimeContext::Create());
        ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
                             CreateRealtimeWriter(realtime_context));
        ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> batch,
                             MakeBatch(wal, /*partitioned=*/false, /*bucket=*/0, row_kinds));
        ASSERT_OK(writer->Write(std::move(batch)));
        ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> progress,
                             writer->PrepareCommitWithProgress(commit_identifier));
        ASSERT_EQ(1, progress.size());
        ASSERT_OK_AND_ASSIGN(int64_t snapshot_id, Commit(progress, commit_identifier));
        ASSERT_OK(writer->RefreshCommittedSnapshot(snapshot_id));
        ASSERT_OK(writer->Close());
        writer.reset();
        realtime_context.reset();

        ASSERT_OK_AND_ASSIGN(std::vector<Row> actual_rows, ReadRows());
        ASSERT_EQ(expected_rows, actual_rows);
    }

    void CheckDropDatePartitionRemovesOffset(bool legacy_partition_name_enabled) {
        fields_ = {arrow::field("id", arrow::int64()), arrow::field("payload", arrow::utf8()),
                   arrow::field("pt", arrow::date32())};
        schema_ = arrow::schema(fields_);
        options_[Options::PARTITION_GENERATE_LEGACY_NAME] =
            legacy_partition_name_enabled ? "true" : "false";
        CreateTable(/*partition_keys=*/{"pt"});
        ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
                             RealtimeContext::Create());
        ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
                             CreateRealtimeWriter(realtime_context));
        constexpr int32_t kDate = 19723;
        constexpr int64_t kRowCount = 3;
        const std::string partition = "2024-01-01";
        ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> batch,
                             MakeDatePartitionBatch(/*first_id=*/0, kRowCount, kDate, partition));
        ASSERT_OK(writer->Write(std::move(batch)));
        ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> commits,
                             writer->PrepareCommitWithProgress(/*commit_identifier=*/0));
        ASSERT_EQ(1, commits.size());
        ASSERT_OK(Commit(commits, /*commit_identifier=*/0));

        const std::string normalized_partition =
            legacy_partition_name_enabled ? std::to_string(kDate) : partition;
        RealtimePartitionBucket partition_bucket({{"pt", normalized_partition}}, /*bucket=*/0);
        ASSERT_OK_AND_ASSIGN(RealtimeOffsetMap offsets_before_drop, ReadCommittedOffsets());
        ASSERT_EQ(1, offsets_before_drop.size());
        ASSERT_EQ(kRowCount, offsets_before_drop.at(partition_bucket));
        ASSERT_OK_AND_ASSIGN(std::shared_ptr<Plan> plan_before_drop,
                             CreatePlan(/*realtime_context=*/nullptr, /*predicate=*/nullptr));
        ASSERT_OK_AND_ASSIGN(int64_t rows_before_drop,
                             CountRows(plan_before_drop, /*realtime_context=*/nullptr));
        ASSERT_EQ(kRowCount, rows_before_drop);

        ASSERT_OK(DropPartition({{"pt", partition}}, /*commit_identifier=*/1));
        ASSERT_OK_AND_ASSIGN(RealtimeOffsetMap offsets_after_drop, ReadCommittedOffsets());
        ASSERT_TRUE(offsets_after_drop.empty());
        ASSERT_OK_AND_ASSIGN(std::shared_ptr<Plan> plan_after_drop,
                             CreatePlan(/*realtime_context=*/nullptr, /*predicate=*/nullptr));
        ASSERT_OK_AND_ASSIGN(int64_t rows_after_drop,
                             CountRows(plan_after_drop, /*realtime_context=*/nullptr));
        ASSERT_EQ(0, rows_after_drop);
        ASSERT_OK(writer->Close());
    }

    std::unique_ptr<UniqueTestDirectory> dir_;
    std::string table_path_;
    std::string commit_user_ = "realtime_commit_user";
    arrow::FieldVector fields_;
    std::shared_ptr<arrow::Schema> schema_;
    std::map<std::string, std::string> options_;
    std::shared_ptr<MemoryPool> pool_;
    mutable std::mutex external_offsets_mutex_;
    mutable std::map<RealtimePartitionBucket, int64_t> next_external_offsets_;
};

TEST_F(RealtimeWriteInteTest, TestRealtimeOperationsRequireEnabledOption) {
    CreateTable(/*partition_keys=*/{});
    std::map<std::string, std::string> disabled_options = options_;
    disabled_options[Options::REALTIME_ENABLED] = "false";
    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
                         RealtimeContext::Create());

    WriteContextBuilder write_builder(table_path_, commit_user_);
    write_builder.SetOptions(disabled_options)
        .WithStreamingMode(true)
        .WithRealtimeContext(realtime_context);
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<WriteContext> write_context, write_builder.Finish());
    ASSERT_NOK_WITH_MSG(FileStoreWrite::Create(std::move(write_context)),
                        "real-time write requires realtime.enabled=true");

    ScanContextBuilder scan_builder(table_path_);
    scan_builder.SetOptions(disabled_options).WithRealtimeContext(realtime_context);
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<ScanContext> scan_context, scan_builder.Finish());
    ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(scan_context)),
                        "real-time scan requires realtime.enabled=true");

    ReadContextBuilder read_builder(table_path_);
    read_builder.SetOptions(disabled_options).WithRealtimeContext(realtime_context);
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<ReadContext> read_context, read_builder.Finish());
    ASSERT_NOK_WITH_MSG(TableRead::Create(std::move(read_context)),
                        "real-time read requires realtime.enabled=true");

    CommitContextBuilder commit_builder(table_path_, commit_user_);
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<CommitContext> commit_context,
                         commit_builder.SetOptions(disabled_options).Finish());
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreCommit> commit,
                         FileStoreCommit::Create(std::move(commit_context)));
    ASSERT_NOK_WITH_MSG(commit->CommitWithProgress(/*realtime_commits=*/{},
                                                   /*commit_identifier=*/0,
                                                   /*watermark=*/std::nullopt),
                        "CommitWithProgress requires realtime.enabled=true");
}

TEST_F(RealtimeWriteInteTest, TestAppendCommitAndRead) {
    CreateTable(/*partition_keys=*/{});
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer, CreateRealtimeWriter());
    std::vector<Row> rows = MakeRows(/*first_id=*/0, /*count=*/10, /*partition=*/"p0");
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> batch,
                         MakeBatch(rows, /*partitioned=*/false));
    ASSERT_OK(writer->Write(std::move(batch)));
    FinalizeCommitAndCheck(writer.get(), /*realtime_commits=*/{}, /*prepare_identifier=*/0, rows);
}

TEST_F(RealtimeWriteInteTest, TestRealtimeDeduplicateRollingRestoresEveryOffsetSidecar) {
    options_[Options::TARGET_FILE_ROW_NUM] = "2";
    ConfigureRealtimeDeduplicateTable();
    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
                         RealtimeContext::Create());
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
                         CreateRealtimeWriter(realtime_context));
    for (int64_t id = 1; id <= 5; id += 2) {
        ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> batch,
                             MakeBatch({Row{id, "old", "p0"}, Row{id + 1, "old", "p0"}},
                                       /*partitioned=*/false));
        ASSERT_OK(writer->Write(std::move(batch)));
    }
    const std::vector<Row> first_expected = {{1, "old", "p0"}, {2, "old", "p0"}, {3, "old", "p0"},
                                             {4, "old", "p0"}, {5, "old", "p0"}, {6, "old", "p0"}};
    ASSERT_OK_AND_ASSIGN(std::vector<Row> rows_before_prepare, ReadRows(realtime_context));
    ASSERT_EQ(first_expected, rows_before_prepare);
    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> progress,
                         writer->PrepareCommitWithProgress(/*commit_identifier=*/0));
    ASSERT_EQ(1, progress.size());
    ASSERT_EQ(OffsetRange(0, 6), progress[0].offset_range);
    std::shared_ptr<CommitMessageImpl> message =
        std::dynamic_pointer_cast<CommitMessageImpl>(progress[0].commit_message);
    ASSERT_NE(nullptr, message);
    const std::vector<std::shared_ptr<DataFileMeta>>& files =
        message->GetNewFilesIncrement().NewFiles();
    ASSERT_EQ(3, files.size());
    for (const std::shared_ptr<DataFileMeta>& file : files) {
        ASSERT_EQ(2, file->row_count);
        ASSERT_EQ(1, std::count(file->extra_files.begin(), file->extra_files.end(),
                                std::optional<std::string>(file->file_name + ".offset")));
    }
    ASSERT_OK_AND_ASSIGN(std::vector<Row> rows_after_prepare, ReadRows(realtime_context));
    ASSERT_EQ(first_expected, rows_after_prepare);
    ASSERT_OK_AND_ASSIGN(int64_t snapshot_id, Commit(progress, /*commit_identifier=*/0));
    ASSERT_OK_AND_ASSIGN(std::vector<Row> rows_before_refresh, ReadRows(realtime_context));
    ASSERT_EQ(first_expected, rows_before_refresh);
    ASSERT_OK(writer->RefreshCommittedSnapshot(snapshot_id));
    ASSERT_OK_AND_ASSIGN(std::vector<Row> rows_after_refresh, ReadRows(realtime_context));
    ASSERT_EQ(first_expected, rows_after_refresh);
    ASSERT_OK(writer->Close());
    writer.reset();
    realtime_context.reset();

    // Restart lookup from disk and overwrite one key from each of the three rolled files.
    ASSERT_OK_AND_ASSIGN(realtime_context, RealtimeContext::Create());
    ASSERT_OK_AND_ASSIGN(writer, CreateRealtimeWriter(realtime_context));
    ASSERT_OK_AND_ASSIGN(std::vector<Row> restored_rows, ReadRows(realtime_context));
    ASSERT_EQ(first_expected, restored_rows);
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> update,
                         MakeBatch({Row{1, "new", "p0"}, Row{3, "new", "p0"}, Row{5, "new", "p0"}},
                                   /*partitioned=*/false));
    ASSERT_OK(writer->Write(std::move(update)));
    const std::vector<Row> expected = {{2, "old", "p0"}, {4, "old", "p0"}, {6, "old", "p0"},
                                       {1, "new", "p0"}, {3, "new", "p0"}, {5, "new", "p0"}};
    ASSERT_OK_AND_ASSIGN(rows_before_prepare, ReadRows(realtime_context));
    ASSERT_EQ(expected, rows_before_prepare);
    ASSERT_OK_AND_ASSIGN(progress, writer->PrepareCommitWithProgress(/*commit_identifier=*/1));
    ASSERT_EQ(1, progress.size());
    ASSERT_EQ(OffsetRange(6, 9), progress[0].offset_range);
    ASSERT_NE(nullptr, progress[0].deleted_offsets);
    ASSERT_EQ(3, progress[0].deleted_offsets->Cardinality());
    for (int64_t offset : {0, 2, 4}) {
        ASSERT_TRUE(progress[0].deleted_offsets->Contains(offset));
    }
    ASSERT_OK_AND_ASSIGN(rows_after_prepare, ReadRows(realtime_context));
    ASSERT_EQ(expected, rows_after_prepare);
    ASSERT_OK_AND_ASSIGN(snapshot_id, Commit(progress, /*commit_identifier=*/1));
    ASSERT_OK_AND_ASSIGN(rows_before_refresh, ReadRows(realtime_context));
    ASSERT_EQ(expected, rows_before_refresh);
    ASSERT_OK(writer->RefreshCommittedSnapshot(snapshot_id));
    ASSERT_OK_AND_ASSIGN(rows_after_refresh, ReadRows(realtime_context));
    ASSERT_EQ(expected, rows_after_refresh);
    ASSERT_OK(writer->Close());
    ASSERT_OK_AND_ASSIGN(std::vector<Row> rows, ReadRows());
    ASSERT_EQ(expected, rows);
}

TEST_F(RealtimeWriteInteTest, TestRefreshPreparationAllowsScanWriteAndStoreRegistration) {
    options_[Options::BUCKET] = "2";
    ConfigureRealtimeDeduplicateTable();
    auto fs = std::make_shared<BlockingFileSystem>(dir_->GetFileSystem());
    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> context, RealtimeContext::Create());
    WriteContextBuilder builder(table_path_, commit_user_);
    builder.SetOptions(options_)
        .WithStreamingMode(true)
        .WithRealtimeContext(context)
        .WithFileSystem(fs);
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<WriteContext> write_context, builder.Finish());
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
                         FileStoreWrite::Create(std::move(write_context)));
    const std::vector<Row> initial = {{1, "old", "p0"}};
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> batch,
                         MakeBatch(initial, /*partitioned=*/false));
    ASSERT_OK(writer->Write(std::move(batch)));
    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> progress,
                         writer->PrepareCommitWithProgress(/*commit_identifier=*/0));
    const auto files = NewFiles(progress);
    ASSERT_EQ(1, files.size());
    ASSERT_OK_AND_ASSIGN(std::shared_ptr<Plan> pinned, CreatePlan(context, /*predicate=*/nullptr));

    // Commit the second bucket from a separate context before this writer registers its store.
    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> seed_context, RealtimeContext::Create());
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> seed_writer,
                         CreateRealtimeWriter(seed_context));
    ASSERT_OK_AND_ASSIGN(batch,
                         MakeBatch({{2, "second", "p0"}}, /*partitioned=*/false, /*bucket=*/1));
    ASSERT_OK(seed_writer->Write(std::move(batch)));
    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> seeded,
                         seed_writer->PrepareCommitWithProgress(/*commit_identifier=*/0));
    progress.insert(progress.end(), seeded.begin(), seeded.end());
    ASSERT_OK_AND_ASSIGN(int64_t snapshot, Commit(progress, /*commit_identifier=*/0));
    ASSERT_OK(seed_writer->Close());

    auto open_gate = fs->BlockNext(
        BlockingFileSystem::Operation::OPEN,
        PathUtil::JoinPath(table_path_, "bucket-0/" + files[0]->file_name + ".offset"));
    auto refreshing = std::async(std::launch::async,
                                 [&]() { return writer->RefreshCommittedSnapshot(snapshot); });
    std::future<Result<std::vector<Row>>> reading;
    std::future<Status> writing;
    std::future<Status> registering;
    ScopeGuard finish([&]() {
        open_gate->Release();
        if (reading.valid()) {
            reading.wait();
        }
        if (writing.valid()) {
            writing.wait();
        }
        if (registering.valid()) {
            registering.wait();
        }
        if (refreshing.valid()) {
            refreshing.wait();
        }
    });
    ASSERT_TRUE(open_gate->WaitForArrival());
    reading = std::async(std::launch::async, [&]() { return ReadRows(context); });
    ASSERT_EQ(std::future_status::ready, reading.wait_for(std::chrono::seconds(5)));
    ASSERT_OK_AND_ASSIGN(std::vector<Row> visible, reading.get());
    std::sort(visible.begin(), visible.end());
    ASSERT_EQ((std::vector<Row>{{1, "old", "p0"}, {2, "second", "p0"}}), visible);
    writing = std::async(std::launch::async, [&]() -> Status {
        PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<RecordBatch> update,
                               MakeBatch({{1, "new", "p0"}}, /*partitioned=*/false));
        return writer->Write(std::move(update));
    });
    ASSERT_EQ(std::future_status::ready, writing.wait_for(std::chrono::seconds(5)));
    ASSERT_OK(writing.get());
    registering = std::async(std::launch::async, [&]() -> Status {
        PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<RecordBatch> update,
                               MakeBatch({{2, "new", "p0"}}, /*partitioned=*/false, /*bucket=*/1));
        return writer->Write(std::move(update));
    });
    ASSERT_EQ(std::future_status::ready, registering.wait_for(std::chrono::seconds(5)));
    ASSERT_OK(registering.get());
    const std::vector<Row> expected = {{1, "new", "p0"}, {2, "new", "p0"}};
    reading = std::async(std::launch::async, [&]() { return ReadRows(context); });
    ASSERT_EQ(std::future_status::ready, reading.wait_for(std::chrono::seconds(5)));
    ASSERT_OK_AND_ASSIGN(visible, reading.get());
    std::sort(visible.begin(), visible.end());
    ASSERT_EQ(expected, visible);

    open_gate->Release();
    ASSERT_OK(refreshing.get());
    ASSERT_OK_AND_ASSIGN(RealtimeMetricValues usage, ReadRealtimeMetrics(writer->GetMetrics()));
    ASSERT_EQ(0, usage.sealed_rows);
    ASSERT_EQ(2, usage.building_rows);
    ASSERT_OK_AND_ASSIGN(visible, ReadRows(pinned, context));
    ASSERT_EQ(initial, visible);
    ASSERT_OK_AND_ASSIGN(visible, ReadRows(context));
    std::sort(visible.begin(), visible.end());
    ASSERT_EQ(expected, visible);

    ASSERT_OK_AND_ASSIGN(progress, writer->PrepareCommitWithProgress(/*commit_identifier=*/1));
    ASSERT_EQ(2, progress.size());
    for (const auto& item : progress) {
        ASSERT_EQ(OffsetRange(1, 2), item.offset_range);
        ASSERT_NE(nullptr, item.deleted_offsets);
        ASSERT_EQ("{0}", item.deleted_offsets->ToString());
    }
    ASSERT_OK_AND_ASSIGN(snapshot, Commit(progress, /*commit_identifier=*/1));
    ASSERT_OK(writer->RefreshCommittedSnapshot(snapshot));
    ASSERT_OK_AND_ASSIGN(visible, ReadRows());
    std::sort(visible.begin(), visible.end());
    ASSERT_EQ(expected, visible);
    ASSERT_OK(writer->Close());
}

TEST_F(RealtimeWriteInteTest, TestRefreshPreparationFailureKeepsProgressAndCanRetry) {
    ConfigureRealtimeDeduplicateTable();
    auto fs = std::make_shared<BlockingFileSystem>(dir_->GetFileSystem());
    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> context, RealtimeContext::Create());
    WriteContextBuilder builder(table_path_, commit_user_);
    builder.SetOptions(options_)
        .WithStreamingMode(true)
        .WithRealtimeContext(context)
        .WithFileSystem(fs);
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<WriteContext> write_context, builder.Finish());
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
                         FileStoreWrite::Create(std::move(write_context)));
    const std::vector<Row> initial = {{1, "old", "p0"}};
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> batch,
                         MakeBatch(initial, /*partitioned=*/false));
    ASSERT_OK(writer->Write(std::move(batch)));
    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> progress,
                         writer->PrepareCommitWithProgress(/*commit_identifier=*/0));
    const auto files = NewFiles(progress);
    ASSERT_EQ(1, files.size());
    ASSERT_OK_AND_ASSIGN(int64_t snapshot, Commit(progress, /*commit_identifier=*/0));
    ASSERT_OK_AND_ASSIGN(RealtimeMetricValues before, ReadRealtimeMetrics(writer->GetMetrics()));
    ASSERT_EQ(1, before.sealed_rows);

    auto open_gate = fs->BlockNext(
        BlockingFileSystem::Operation::OPEN,
        PathUtil::JoinPath(table_path_, "bucket-0/" + files[0]->file_name + ".offset"));
    open_gate->Release(Status::IOError("injected sidecar open failure"));
    ASSERT_NOK_WITH_MSG(writer->RefreshCommittedSnapshot(snapshot),
                        "injected sidecar open failure");
    ASSERT_OK_AND_ASSIGN(RealtimeMetricValues failed, ReadRealtimeMetrics(writer->GetMetrics()));
    ASSERT_EQ(before.sealed_rows, failed.sealed_rows);
    ASSERT_EQ(before.total_memory, failed.total_memory);
    ASSERT_OK_AND_ASSIGN(std::vector<Row> visible, ReadRows(context));
    ASSERT_EQ(initial, visible);

    ASSERT_OK(writer->RefreshCommittedSnapshot(snapshot));
    ASSERT_OK_AND_ASSIGN(RealtimeMetricValues after, ReadRealtimeMetrics(writer->GetMetrics()));
    ASSERT_EQ(0, after.sealed_rows);
    ASSERT_EQ(0, after.total_rows);
    ASSERT_OK_AND_ASSIGN(visible, ReadRows(context));
    ASSERT_EQ(initial, visible);
    ASSERT_OK_AND_ASSIGN(batch, MakeBatch({{1, "new", "p0"}}, /*partitioned=*/false));
    ASSERT_OK(writer->Write(std::move(batch)));
    ASSERT_OK_AND_ASSIGN(progress, writer->PrepareCommitWithProgress(/*commit_identifier=*/1));
    ASSERT_EQ(1, progress.size());
    ASSERT_EQ(OffsetRange(1, 2), progress[0].offset_range);
    ASSERT_NE(nullptr, progress[0].deleted_offsets);
    ASSERT_EQ("{0}", progress[0].deleted_offsets->ToString());
    ASSERT_OK_AND_ASSIGN(snapshot, Commit(progress, /*commit_identifier=*/1));
    ASSERT_OK(writer->RefreshCommittedSnapshot(snapshot));
    ASSERT_OK_AND_ASSIGN(visible, ReadRows());
    ASSERT_EQ((std::vector<Row>{{1, "new", "p0"}}), visible);
    ASSERT_OK(writer->Close());
}

TEST_F(RealtimeWriteInteTest, TestRealtimeDeduplicateWritesDeletionVector) {
    ConfigureRealtimeDeduplicateTable();

    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
                         RealtimeContext::Create());
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
                         CreateRealtimeWriter(realtime_context));

    const std::vector<Row> first_rows = {{1, "old-one", "p0"}, {2, "two", "p0"}};
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> first_batch,
                         MakeBatch(first_rows, /*partitioned=*/false));
    ASSERT_OK(writer->Write(std::move(first_batch)));
    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> first_progress,
                         writer->PrepareCommitWithProgress(/*commit_identifier=*/0));
    ASSERT_EQ(1, first_progress.size());
    ASSERT_EQ(OffsetRange(0, 2), first_progress[0].offset_range);
    std::shared_ptr<CommitMessageImpl> first_message =
        std::dynamic_pointer_cast<CommitMessageImpl>(first_progress[0].commit_message);
    ASSERT_NE(nullptr, first_message);
    ASSERT_EQ(1, first_message->GetNewFilesIncrement().NewFiles().size());
    ASSERT_TRUE(first_message->GetCompactIncrement().NewIndexFiles().empty());
    ASSERT_OK_AND_ASSIGN(int64_t first_snapshot_id,
                         Commit(first_progress, /*commit_identifier=*/0));

    ASSERT_OK_AND_ASSIGN(std::vector<Row> first_rows_before_refresh, ReadRows(realtime_context));
    ASSERT_EQ(first_rows, first_rows_before_refresh);

    ASSERT_OK(writer->RefreshCommittedSnapshot(first_snapshot_id));

    ASSERT_OK_AND_ASSIGN(std::vector<Row> first_rows_after_refresh, ReadRows(realtime_context));
    ASSERT_EQ(first_rows, first_rows_after_refresh);

    const std::vector<Row> second_rows = {{1, "new-one", "p0"}, {3, "three", "p0"}};
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> second_batch,
                         MakeBatch(second_rows, /*partitioned=*/false));
    ASSERT_OK(writer->Write(std::move(second_batch)));

    const std::vector<Row> expected_rows = {
        {2, "two", "p0"}, {1, "new-one", "p0"}, {3, "three", "p0"}};
    ASSERT_OK_AND_ASSIGN(std::vector<Row> realtime_rows, ReadRows(realtime_context));
    ASSERT_EQ(expected_rows, realtime_rows);

    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> second_progress,
                         writer->PrepareCommitWithProgress(/*commit_identifier=*/1));
    ASSERT_EQ(1, second_progress.size());
    ASSERT_EQ(OffsetRange(2, 4), second_progress[0].offset_range);
    std::shared_ptr<CommitMessageImpl> second_message =
        std::dynamic_pointer_cast<CommitMessageImpl>(second_progress[0].commit_message);
    ASSERT_NE(nullptr, second_message);
    ASSERT_EQ(1, second_message->GetNewFilesIncrement().NewFiles().size());
    ASSERT_TRUE(second_message->GetCompactIncrement().NewIndexFiles().empty());
    ASSERT_NE(nullptr, second_progress[0].deleted_offsets);
    ASSERT_FALSE(second_progress[0].deleted_offsets->IsEmpty());

    ASSERT_OK_AND_ASSIGN(int64_t second_snapshot_id,
                         Commit(second_progress, /*commit_identifier=*/1));

    ASSERT_OK_AND_ASSIGN(std::vector<Row> rows_before_refresh, ReadRows(realtime_context));
    ASSERT_EQ(expected_rows, rows_before_refresh);

    ASSERT_OK(writer->RefreshCommittedSnapshot(second_snapshot_id));

    ASSERT_OK_AND_ASSIGN(std::vector<Row> rows_after_refresh, ReadRows(realtime_context));
    ASSERT_EQ(expected_rows, rows_after_refresh);
    ASSERT_OK(writer->Close());
    writer.reset();
    realtime_context.reset();

    ASSERT_OK_AND_ASSIGN(std::vector<Row> disk_rows, ReadRows());
    ASSERT_EQ(expected_rows, disk_rows);
}

TEST_F(RealtimeWriteInteTest, TestRealtimeDeduplicateRestoresHistoricalSchemas) {
    ConfigureRealtimeDeduplicateTable();
    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> context, RealtimeContext::Create());
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer, CreateRealtimeWriter(context));
    ASSERT_OK(WriteCommitAndRefresh({{1, "old-one", "p0"}, {2, "old-two", "p0"}},
                                    /*partitioned=*/false, /*bucket=*/0, {}, 0, writer.get()));
    ASSERT_OK(WriteCommitAndRefresh({{3, "three", "p0"}}, /*partitioned=*/false,
                                    /*bucket=*/0, {}, 1, writer.get()));
    ASSERT_OK(writer->Close());
    writer.reset();
    context.reset();

    // Keep field IDs while renaming the deduplicate key and a regular column, moving the offset
    // column, and adding a nullable column. Restore old files and their .offset sidecars by ID.
    const auto renamed_key = fields_[1]->WithName("renamed_id");
    const auto renamed_payload = fields_[2]->WithName("renamed_payload");
    const auto extra = arrow::field("extra", arrow::utf8());
    options_[Options::REALTIME_DEDUPLICATE_KEY_FIELDS] = renamed_key->name();
    options_[Options::BUCKET_KEY] = renamed_key->name();
    ASSERT_OK(TestHelper::WriteNextSchema(
        dir_->GetFileSystem(), table_path_,
        {DataField(1, renamed_key), DataField(2, renamed_payload), DataField(3, fields_[3]),
         DataField(0, fields_[0]), DataField(4, extra)},
        /*highest_field_id=*/4, options_));
    fields_ = {renamed_key, renamed_payload, fields_[3], fields_[0], extra};
    schema_ = arrow::schema(fields_);
    ASSERT_OK_AND_ASSIGN(context, RealtimeContext::Create());
    ASSERT_OK_AND_ASSIGN(writer, CreateRealtimeWriter(context));
    auto array = arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(schema_->fields()), R"([
                         [1, "new-one", "p0", 3, "x"],
                         [2, "deleted", "p0", 4, null],
                         [4, "four", "p0", 5, "y"]
                     ])")
                     .ValueOrDie();
    ArrowArray c_array;
    ASSERT_TRUE(arrow::ExportArray(*array, &c_array).ok());
    ASSERT_OK_AND_ASSIGN(
        std::unique_ptr<RecordBatch> batch,
        RecordBatchBuilder(&c_array)
            .SetBucket(0)
            .SetRowKinds({RecordBatch::RowKind::INSERT, RecordBatch::RowKind::DELETE,
                          RecordBatch::RowKind::INSERT})
            .Finish());
    ASSERT_OK(writer->Write(std::move(batch)));

    const auto projection = arrow::schema({fields_[0], fields_[1], fields_[4]});
    const std::string expected = R"([
        [0, 3, "three", null],
        [0, 1, "new-one", "x"],
        [0, 4, "four", "y"]
    ])";
    ASSERT_OK_AND_ASSIGN(std::shared_ptr<Plan> plan, CreatePlan(context, /*predicate=*/nullptr));
    ReadPlanWithSchemaAndCheck(plan, context, projection, expected);
    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> progress,
                         writer->PrepareCommitWithProgress(2));
    ASSERT_OK_AND_ASSIGN(int64_t snapshot, Commit(progress, 2));
    ASSERT_OK(writer->RefreshCommittedSnapshot(snapshot));
    ASSERT_OK(writer->Close());
    writer.reset();
    context.reset();

    // Persisted file-position DVs must still hide the overwritten/deleted historical rows.
    ASSERT_OK_AND_ASSIGN(context, RealtimeContext::Create());
    ASSERT_OK_AND_ASSIGN(writer, CreateRealtimeWriter(context));
    ASSERT_OK_AND_ASSIGN(plan, CreatePlan(context, /*predicate=*/nullptr));
    ReadPlanWithSchemaAndCheck(plan, context, projection, expected);

    ASSERT_OK_AND_ASSIGN(Snapshot compact_snapshot, CompactAndCommit(/*partition=*/{}, /*bucket=*/0,
                                                                     /*commit_identifier=*/3));
    ASSERT_EQ(Snapshot::CommitKind::Compact(), compact_snapshot.GetCommitKind());
    ASSERT_OK(writer->RefreshCommittedSnapshot(compact_snapshot.Id()));
    ASSERT_OK_AND_ASSIGN(plan, CreatePlan(context, /*predicate=*/nullptr));
    ReadPlanWithSchemaAndCheck(plan, context, projection, expected);
    ASSERT_OK(writer->Close());
}

TEST_F(RealtimeWriteInteTest, TestRealtimeDeduplicateCommitRetryMergesLatestDv) {
    ConfigureRealtimeDeduplicateTable();
    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
                         RealtimeContext::Create());
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
                         CreateRealtimeWriter(realtime_context));
    ASSERT_OK_AND_ASSIGN(
        std::unique_ptr<RecordBatch> initial_batch,
        MakeBatch({Row{1, "old-one", "p0"}, Row{2, "two", "p0"}, Row{3, "three", "p0"}},
                  /*partitioned=*/false));
    ASSERT_OK(writer->Write(std::move(initial_batch)));
    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> initial_progress,
                         writer->PrepareCommitWithProgress(/*commit_identifier=*/0));
    ASSERT_EQ(1, initial_progress.size());
    ASSERT_NE(nullptr, initial_progress[0].deleted_offsets);
    ASSERT_TRUE(initial_progress[0].deleted_offsets->IsEmpty());
    ASSERT_OK_AND_ASSIGN(int64_t initial_snapshot,
                         Commit(initial_progress, /*commit_identifier=*/0));
    ASSERT_OK(writer->RefreshCommittedSnapshot(initial_snapshot));

    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> update_batch,
                         MakeBatch({Row{1, "new-one", "p0"}}, /*partitioned=*/false));
    ASSERT_OK(writer->Write(std::move(update_batch)));
    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> progress,
                         writer->PrepareCommitWithProgress(/*commit_identifier=*/1));
    ASSERT_EQ(1, progress.size());
    ASSERT_NE(nullptr, progress[0].deleted_offsets);
    ASSERT_EQ("{0}", progress[0].deleted_offsets->ToString());
    auto initial_message =
        std::dynamic_pointer_cast<CommitMessageImpl>(initial_progress[0].commit_message);
    auto update_message = std::dynamic_pointer_cast<CommitMessageImpl>(progress[0].commit_message);
    ASSERT_NE(nullptr, initial_message);
    ASSERT_NE(nullptr, update_message);
    ASSERT_TRUE(update_message->GetCompactIncrement().IsEmpty());
    std::vector<RealtimeCommitProgress> missing_deletions = progress;
    missing_deletions[0].deleted_offsets.reset();
    ASSERT_NOK_WITH_MSG(Commit(missing_deletions, /*commit_identifier=*/1),
                        "requires a deleted-offset bitmap");
    const std::vector<std::shared_ptr<DataFileMeta>> initial_files = NewFiles(initial_progress);
    ASSERT_EQ(1, initial_files.size());

    // A concurrent physical DV update deletes a different row from the same file and bucket.
    auto index_paths =
        std::make_shared<MockIndexPathFactory>(PathUtil::JoinPath(table_path_, "index"));
    DeletionVectorsIndexFile dv_file(dir_->GetFileSystem(), index_paths, /*bitmap64=*/false, pool_);
    RoaringBitmap32 deleted_positions;
    deleted_positions.Add(1);
    ASSERT_OK_AND_ASSIGN(
        std::shared_ptr<IndexFileMeta> concurrent_dv,
        dv_file.WriteSingleFile({{initial_files[0]->file_name,
                                  std::make_shared<BitmapDeletionVector>(deleted_positions)}}));
    auto concurrent_message = std::make_shared<CommitMessageImpl>(
        initial_message->Partition(), /*bucket=*/0, /*total_buckets=*/1,
        DataIncrement({}, {}, {}, {concurrent_dv}, {}), CompactIncrement({}, {}, {}, {}, {}));
    CommitContextBuilder concurrent_builder(table_path_, "concurrent-dv-writer");
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<CommitContext> concurrent_context,
                         concurrent_builder.SetOptions(options_).Finish());
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreCommit> concurrent_commit,
                         FileStoreCommit::Create(std::move(concurrent_context)));
    bool interleaved = false;
    auto fs = std::make_shared<InterleavingSnapshotFileSystem>(
        PathUtil::JoinPath(table_path_, "snapshot/snapshot-2"), [&]() -> Status {
            interleaved = true;
            return concurrent_commit->Commit({concurrent_message}, /*commit_identifier=*/0,
                                             /*watermark=*/std::nullopt);
        });
    CommitContextBuilder builder(table_path_, commit_user_);
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<CommitContext> context,
                         builder.SetOptions(options_).WithFileSystem(fs).Finish());
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreCommit> commit,
                         FileStoreCommit::Create(std::move(context)));
    ASSERT_OK_AND_ASSIGN(
        int64_t snapshot_id,
        commit->CommitWithProgress(progress, /*commit_identifier=*/1, /*watermark=*/std::nullopt));
    ASSERT_TRUE(interleaved);
    ASSERT_EQ(3, snapshot_id);
    ASSERT_OK_AND_ASSIGN(uint64_t attempts, commit->GetCommitMetrics()->GetCounter(
                                                CommitMetrics::LAST_COMMIT_ATTEMPTS));
    ASSERT_EQ(2, attempts);
    ASSERT_EQ("{0}", progress[0].deleted_offsets->ToString());
    const std::vector<Row> expected_rows = {{3, "three", "p0"}, {1, "new-one", "p0"}};
    ASSERT_OK_AND_ASSIGN(std::vector<Row> disk_rows, ReadRows());
    ASSERT_EQ(expected_rows, disk_rows);
    ASSERT_OK_AND_ASSIGN(std::vector<Row> rows_before_refresh, ReadRows(realtime_context));
    ASSERT_EQ(expected_rows, rows_before_refresh);
    ASSERT_OK(writer->RefreshCommittedSnapshot(snapshot_id));
    ASSERT_OK_AND_ASSIGN(std::vector<Row> rows_after_refresh, ReadRows(realtime_context));
    ASSERT_EQ(expected_rows, rows_after_refresh);
    ASSERT_OK_AND_ASSIGN(
        int64_t retried_snapshot,
        commit->CommitWithProgress(progress, /*commit_identifier=*/1, /*watermark=*/std::nullopt));
    ASSERT_EQ(snapshot_id, retried_snapshot);
    ASSERT_OK(writer->Close());
}

TEST_F(RealtimeWriteInteTest, TestRealtimeDeduplicateCommitRetryRemapsCompactedOffsets) {
    ConfigureRealtimeDeduplicateTable();
    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
                         RealtimeContext::Create());
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
                         CreateRealtimeWriter(realtime_context));
    for (int64_t id = 1; id <= 2; ++id) {
        ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> batch,
                             MakeBatch({Row{id, "value", "p0"}}, /*partitioned=*/false));
        ASSERT_OK(writer->Write(std::move(batch)));
        ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> progress,
                             writer->PrepareCommitWithProgress(/*commit_identifier=*/id - 1));
        ASSERT_OK_AND_ASSIGN(int64_t snapshot, Commit(progress, /*commit_identifier=*/id - 1));
        ASSERT_OK(writer->RefreshCommittedSnapshot(snapshot));
    }
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> delete_batch,
                         MakeBatch({Row{2, "ignored", "p0"}}, /*partitioned=*/false, /*bucket=*/0,
                                   {RecordBatch::RowKind::DELETE}));
    ASSERT_OK(writer->Write(std::move(delete_batch)));
    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> progress,
                         writer->PrepareCommitWithProgress(/*commit_identifier=*/2));
    ASSERT_EQ(1, progress.size());
    ASSERT_NE(nullptr, progress[0].deleted_offsets);
    ASSERT_EQ("{1}", progress[0].deleted_offsets->ToString());
    bool interleaved = false;
    auto fs = std::make_shared<InterleavingSnapshotFileSystem>(
        PathUtil::JoinPath(table_path_, "snapshot/snapshot-3"), [&]() -> Status {
            PAIMON_ASSIGN_OR_RAISE(
                Snapshot compacted,
                CompactAndCommit(/*partition=*/{}, /*bucket=*/0, /*commit_identifier=*/1));
            if (compacted.Id() != 3 ||
                compacted.GetCommitKind() != Snapshot::CommitKind::Compact()) {
                return Status::Invalid("expected competing compaction snapshot");
            }
            interleaved = true;
            return Status::OK();
        });
    CommitContextBuilder builder(table_path_, commit_user_);
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<CommitContext> context,
                         builder.SetOptions(options_).WithFileSystem(fs).Finish());
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreCommit> commit,
                         FileStoreCommit::Create(std::move(context)));
    ASSERT_OK_AND_ASSIGN(
        int64_t snapshot_id,
        commit->CommitWithProgress(progress, /*commit_identifier=*/2, /*watermark=*/std::nullopt));
    ASSERT_TRUE(interleaved);
    ASSERT_EQ(4, snapshot_id);
    ASSERT_OK_AND_ASSIGN(uint64_t attempts, commit->GetCommitMetrics()->GetCounter(
                                                CommitMetrics::LAST_COMMIT_ATTEMPTS));
    ASSERT_EQ(2, attempts);
    ASSERT_EQ("{1}", progress[0].deleted_offsets->ToString());
    ASSERT_OK_AND_ASSIGN(std::vector<Row> disk_rows, ReadRows());
    ASSERT_EQ((std::vector<Row>{{1, "value", "p0"}}), disk_rows);
    ASSERT_OK(writer->Close());
}

class RealtimeDeduplicateCompactionInteTest
    : public RealtimeWriteInteTest,
      public ::testing::WithParamInterface<std::tuple<bool, bool>> {};

TEST_P(RealtimeDeduplicateCompactionInteTest, TestCommitTransfersLatestDvToCompactedFiles) {
    const auto [force_retry, prepare_between_rounds] = GetParam();
    options_[Options::COMPACTION_FORCE_REWRITE_ALL_FILES] = "true";
    options_[Options::FILE_INDEX_IN_MANIFEST_THRESHOLD] = "1B";
    ConfigureRealtimeDeduplicateTable();
    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
                         RealtimeContext::Create());
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
                         CreateRealtimeWriter(realtime_context));
    // Establish D0 before compaction: offset 0 must not appear in either rewrite's output.
    const std::vector<std::vector<Row>> batches = {
        {{1, "one", "p0"}, {2, "two", "p0"}, {3, "three", "p0"}},
        {{4, "four", "p0"}, {5, "five", "p0"}},
        {{1, "ignored", "p0"}}};
    for (int64_t identifier = 0; identifier < 3; ++identifier) {
        const std::vector<RecordBatch::RowKind> kinds =
            identifier == 2 ? std::vector<RecordBatch::RowKind>{RecordBatch::RowKind::DELETE}
                            : std::vector<RecordBatch::RowKind>{};
        ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> batch,
                             MakeBatch(batches[identifier], /*partitioned=*/false, /*bucket=*/0,
                                       /*row_kinds=*/kinds));
        ASSERT_OK(writer->Write(std::move(batch)));
        ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> progress,
                             writer->PrepareCommitWithProgress(/*commit_identifier=*/identifier));
        ASSERT_OK_AND_ASSIGN(int64_t snapshot, Commit(progress, /*commit_identifier=*/identifier));
        ASSERT_OK(writer->RefreshCommittedSnapshot(snapshot));
    }

    WriteContextBuilder write_builder(table_path_, "compactor");
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<WriteContext> write_context,
                         write_builder.SetOptions(options_).WithStreamingMode(true).Finish());
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> compactor,
                         FileStoreWrite::Create(std::move(write_context)));
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> forbidden_batch,
                         MakeBatch({{99, "not-written", "p0"}}, /*partitioned=*/false));
    ASSERT_NOK_WITH_MSG(compactor->Write(std::move(forbidden_batch)), "only supports compaction");
    ResetExternalOffset(/*partition=*/{}, /*bucket=*/0, /*next_offset=*/6);
    // F1/F2 -> F3 -> F4, with no snapshot publication between the two rewrites.
    std::vector<std::shared_ptr<CommitMessage>> messages;
    for (int32_t round = 0; round < 2; ++round) {
        ASSERT_OK(compactor->Compact(/*partition=*/{}, /*bucket=*/0, /*full_compaction=*/true));
        if (prepare_between_rounds || round == 1) {
            ASSERT_OK_AND_ASSIGN(
                std::vector<std::shared_ptr<CommitMessage>> prepared,
                compactor->PrepareCommit(/*wait_compaction=*/true, /*commit_identifier=*/0));
            messages.insert(messages.end(), prepared.begin(), prepared.end());
        }
    }
    ManifestEntryChanges changes(1, /*drop_delete_file_stats=*/true);
    for (const auto& message : messages) {
        ASSERT_OK(changes.Collect(message));
    }
    ASSERT_OK_AND_ASSIGN(RealtimeCompactionPlan compact_plan,
                         RealtimeCompactionPlan::Create(changes.compact_table_files));
    ASSERT_EQ(1, compact_plan.intermediate_files.size());
    std::shared_ptr<DataFileMeta> final_output;
    for (const ManifestEntry& entry : compact_plan.delta_files) {
        if (entry.Kind() == FileKind::Add()) {
            ASSERT_EQ(nullptr, final_output);
            final_output = entry.File();
        }
    }
    ASSERT_NE(nullptr, final_output);
    ASSERT_EQ(4, final_output->row_count);
    ASSERT_FALSE(final_output->extra_files.empty());

    // D1 is published against compact-before AFTER both rewrites have completed.
    ASSERT_OK_AND_ASSIGN(
        std::unique_ptr<RecordBatch> update_batch,
        MakeBatch(
            {{2, "updated", "p0"}, {3, "ignored", "p0"}}, /*partitioned=*/false, /*bucket=*/0,
            /*row_kinds=*/{RecordBatch::RowKind::UPDATE_AFTER, RecordBatch::RowKind::DELETE}));
    ASSERT_OK(writer->Write(std::move(update_batch)));
    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> update_progress,
                         writer->PrepareCommitWithProgress(/*commit_identifier=*/3));
    ASSERT_OK_AND_ASSIGN(int64_t update_snapshot, Commit(update_progress, /*commit_identifier=*/3));
    ASSERT_EQ(4, update_snapshot);
    ASSERT_OK(writer->RefreshCommittedSnapshot(update_snapshot));

    // D2 also deletes from a non-compacted file. Its DV must survive bucket replacement.
    ASSERT_OK_AND_ASSIGN(
        std::unique_ptr<RecordBatch> concurrent_batch,
        MakeBatch(
            {{2, "latest", "p0"}, {4, "ignored", "p0"}}, /*partitioned=*/false, /*bucket=*/0,
            /*row_kinds=*/{RecordBatch::RowKind::UPDATE_AFTER, RecordBatch::RowKind::DELETE}));
    ASSERT_OK(writer->Write(std::move(concurrent_batch)));
    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> concurrent_progress,
                         writer->PrepareCommitWithProgress(/*commit_identifier=*/4));
    bool interleaved = false;
    auto publish_build = [&]() -> Status {
        PAIMON_ASSIGN_OR_RAISE(int64_t snapshot,
                               Commit(concurrent_progress, /*commit_identifier=*/4));
        if (snapshot != 5) {
            return Status::Invalid("expected interleaving build snapshot 5");
        }
        interleaved = true;
        return Status::OK();
    };
    std::shared_ptr<FileSystem> fs = dir_->GetFileSystem();
    if (force_retry) {
        fs = std::make_shared<InterleavingSnapshotFileSystem>(
            PathUtil::JoinPath(table_path_, "snapshot/snapshot-5"), publish_build);
    } else {
        ASSERT_OK(publish_build());
    }
    CommitContextBuilder commit_builder(table_path_, "compactor");
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<CommitContext> commit_context,
                         commit_builder.SetOptions(options_).WithFileSystem(fs).Finish());
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreCommit> commit,
                         FileStoreCommit::Create(std::move(commit_context)));
    ASSERT_OK(commit->Commit(messages, /*commit_identifier=*/0));
    ASSERT_TRUE(interleaved);
    // Only unpublished intermediate outputs and their sidecars are reclaimed after publication.
    const std::string bucket_path = PathUtil::JoinPath(table_path_, "bucket-0");
    const auto& intermediate = compact_plan.intermediate_files[0].File();
    ASSERT_EQ(2, intermediate->extra_files.size());
    std::vector<std::pair<std::shared_ptr<DataFileMeta>, bool>> expected_files = {
        {intermediate, /*exists_after_commit=*/false},
        {final_output, /*exists_after_commit=*/true}};
    // Original inputs and their sidecars are still referenced by historical snapshots.
    for (const ManifestEntry& entry : compact_plan.delta_files) {
        if (entry.Kind() == FileKind::Delete()) {
            ASSERT_EQ(2, entry.File()->extra_files.size());
            expected_files.emplace_back(entry.File(), /*exists_after_commit=*/true);
        }
    }
    ASSERT_EQ(4, expected_files.size());
    for (const auto& [file, expected_exists] : expected_files) {
        ASSERT_OK_AND_ASSIGN(bool exists, dir_->GetFileSystem()->Exists(
                                              PathUtil::JoinPath(bucket_path, file->file_name)));
        ASSERT_EQ(expected_exists, exists);
        for (const auto& extra : file->extra_files) {
            ASSERT_TRUE(extra);
            ASSERT_OK_AND_ASSIGN(
                exists, dir_->GetFileSystem()->Exists(PathUtil::JoinPath(bucket_path, *extra)));
            ASSERT_EQ(expected_exists, exists);
        }
    }
    ASSERT_OK_AND_ASSIGN(uint64_t attempts, commit->GetCommitMetrics()->GetCounter(
                                                CommitMetrics::LAST_COMMIT_ATTEMPTS));
    ASSERT_EQ(force_retry ? 2 : 1, attempts);
    const std::vector<Row> expected = {{5, "five", "p0"}, {2, "latest", "p0"}};
    ASSERT_OK_AND_ASSIGN(std::vector<Row> disk_rows, ReadRows());
    ASSERT_EQ(expected, disk_rows);
    ASSERT_OK_AND_ASSIGN(std::vector<Row> realtime_rows, ReadRows(realtime_context));
    ASSERT_EQ(expected, realtime_rows);

    // Prove migration, not an implicit re-rewrite: F4 is unchanged, with a physical DV {0,1,2}.
    ASSERT_OK_AND_ASSIGN(std::shared_ptr<Plan> plan,
                         CreatePlan(/*realtime_context=*/nullptr, /*predicate=*/nullptr));
    bool checked_output = false;
    for (const auto& split : plan->Splits()) {
        auto data = std::dynamic_pointer_cast<DataSplitImpl>(split);
        ASSERT_NE(nullptr, data);
        for (size_t i = 0; i < data->DataFiles().size(); ++i) {
            if (data->DataFiles()[i]->file_name != final_output->file_name) {
                continue;
            }
            checked_output = true;
            ASSERT_EQ(4, data->DataFiles()[i]->row_count);
            ASSERT_EQ(data->DataFiles().size(), data->DeletionFiles().size());
            ASSERT_TRUE(data->DeletionFiles()[i]);
            ASSERT_OK_AND_ASSIGN(
                PAIMON_UNIQUE_PTR<DeletionVector> dv,
                DeletionVector::Read(fs.get(), *data->DeletionFiles()[i], pool_.get()));
            ASSERT_OK_AND_ASSIGN(int64_t count, dv->GetCardinality());
            ASSERT_EQ(3, count);
            for (int64_t position = 0; position < 4; ++position) {
                ASSERT_OK_AND_ASSIGN(bool deleted, dv->IsDeleted(position));
                ASSERT_EQ(position < 3, deleted);
            }
        }
    }
    ASSERT_TRUE(checked_output);
    ASSERT_OK(writer->RefreshCommittedSnapshot(6));
    ASSERT_OK_AND_ASSIGN(realtime_rows, ReadRows(realtime_context));
    ASSERT_EQ(expected, realtime_rows);
    ASSERT_OK(writer->Close());
    ASSERT_OK(compactor->Close());

    // Restart uses the newly compacted files' key->offset sidecars and keeps existing DVs.
    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> restarted_context,
                         RealtimeContext::Create());
    ASSERT_OK_AND_ASSIGN(writer, CreateRealtimeWriter(restarted_context));
    ASSERT_OK_AND_ASSIGN(realtime_rows, ReadRows(restarted_context));
    ASSERT_EQ(expected, realtime_rows);
    ASSERT_OK_AND_ASSIGN(disk_rows, ReadRows());
    ASSERT_EQ(expected, disk_rows);
    const std::vector<Row> expected_after_delete = {{2, "latest", "p0"}};
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> delete_batch,
                         MakeBatch({{5, "ignored", "p0"}}, /*partitioned=*/false, /*bucket=*/0,
                                   /*row_kinds=*/{RecordBatch::RowKind::DELETE}));
    ASSERT_OK(writer->Write(std::move(delete_batch)));
    // The buffered deletion is visible in realtime, but is not durable yet.
    ASSERT_OK_AND_ASSIGN(realtime_rows, ReadRows(restarted_context));
    ASSERT_EQ(expected_after_delete, realtime_rows);
    ASSERT_OK_AND_ASSIGN(disk_rows, ReadRows());
    ASSERT_EQ(expected, disk_rows);
    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> delete_progress,
                         writer->PrepareCommitWithProgress(/*commit_identifier=*/5));
    ASSERT_OK_AND_ASSIGN(realtime_rows, ReadRows(restarted_context));
    ASSERT_EQ(expected_after_delete, realtime_rows);
    ASSERT_OK_AND_ASSIGN(disk_rows, ReadRows());
    ASSERT_EQ(expected, disk_rows);
    ASSERT_OK_AND_ASSIGN(int64_t snapshot, Commit(delete_progress, /*commit_identifier=*/5));
    ASSERT_EQ(7, snapshot);
    // Publication must preserve realtime visibility even before the writer acknowledges it.
    ASSERT_OK_AND_ASSIGN(realtime_rows, ReadRows(restarted_context));
    ASSERT_EQ(expected_after_delete, realtime_rows);
    ASSERT_OK_AND_ASSIGN(disk_rows, ReadRows());
    ASSERT_EQ(expected_after_delete, disk_rows);
    ASSERT_OK(writer->RefreshCommittedSnapshot(snapshot));
    ASSERT_OK_AND_ASSIGN(realtime_rows, ReadRows(restarted_context));
    ASSERT_EQ(expected_after_delete, realtime_rows);
    ASSERT_OK_AND_ASSIGN(disk_rows, ReadRows());
    ASSERT_EQ(expected_after_delete, disk_rows);
    ASSERT_OK(writer->Close());
}

INSTANTIATE_TEST_SUITE_P(Compaction, RealtimeDeduplicateCompactionInteTest,
                         ::testing::Combine(::testing::Bool(), ::testing::Bool()));

TEST_F(RealtimeWriteInteTest, TestRealtimeDeduplicateCompactionRejectsReplacedInputs) {
    ConfigureRealtimeDeduplicateTable();
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer, CreateRealtimeWriter());
    const std::vector<Row> expected = {{1, "one", "p0"}, {2, "two", "p0"}};
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> batch,
                         MakeBatch(expected, /*partitioned=*/false));
    ASSERT_OK(writer->Write(std::move(batch)));
    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> progress,
                         writer->PrepareCommitWithProgress(/*commit_identifier=*/0));
    ASSERT_OK_AND_ASSIGN(int64_t initial_snapshot, Commit(progress, /*commit_identifier=*/0));
    ASSERT_EQ(1, initial_snapshot);
    ASSERT_OK(writer->Close());

    WriteContextBuilder write_builder(table_path_, "stale-compactor");
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<WriteContext> write_context,
                         write_builder.SetOptions(options_).WithStreamingMode(true).Finish());
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> compactor,
                         FileStoreWrite::Create(std::move(write_context)));
    ASSERT_OK(compactor->Compact(/*partition=*/{}, /*bucket=*/0, /*full_compaction=*/true));
    ASSERT_OK_AND_ASSIGN(
        std::vector<std::shared_ptr<CommitMessage>> messages,
        compactor->PrepareCommit(/*wait_compaction=*/true, /*commit_identifier=*/0));
    ASSERT_EQ(1, messages.size());
    auto message = std::dynamic_pointer_cast<CommitMessageImpl>(messages[0]);
    ASSERT_NE(nullptr, message);
    ASSERT_EQ(1, message->GetCompactIncrement().CompactAfter().size());

    ASSERT_OK_AND_ASSIGN(Snapshot replacement,
                         CompactAndCommit(/*partition=*/{}, /*bucket=*/0, /*commit_identifier=*/1));
    ASSERT_EQ(2, replacement.Id());
    CommitContextBuilder commit_builder(table_path_, "stale-compactor");
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<CommitContext> commit_context,
                         commit_builder.SetOptions(options_).Finish());
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreCommit> commit,
                         FileStoreCommit::Create(std::move(commit_context)));
    ASSERT_NOK_WITH_MSG(commit->Commit(messages, /*commit_identifier=*/0),
                        "input is no longer active");
    ASSERT_OK_AND_ASSIGN(std::vector<Row> rows, ReadRows());
    ASSERT_EQ(expected, rows);
    // A known rejected commit can be aborted without damaging the winning compaction.
    ASSERT_OK(commit->Abort(messages));
    for (const auto& file : message->GetCompactIncrement().CompactAfter()) {
        const std::string bucket_path = PathUtil::JoinPath(table_path_, "bucket-0");
        ASSERT_OK_AND_ASSIGN(bool exists, dir_->GetFileSystem()->Exists(
                                              PathUtil::JoinPath(bucket_path, file->file_name)));
        ASSERT_FALSE(exists);
        for (const auto& extra : file->extra_files) {
            ASSERT_TRUE(extra);
            ASSERT_OK_AND_ASSIGN(
                exists, dir_->GetFileSystem()->Exists(PathUtil::JoinPath(bucket_path, *extra)));
            ASSERT_FALSE(exists);
        }
    }
    ASSERT_OK_AND_ASSIGN(rows, ReadRows());
    ASSERT_EQ(expected, rows);
    ASSERT_OK(compactor->Close());
}

TEST_F(RealtimeWriteInteTest, TestRealtimeDeduplicateCompactionWithAllRowsDeleted) {
    ConfigureRealtimeDeduplicateTable();
    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
                         RealtimeContext::Create());
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
                         CreateRealtimeWriter(realtime_context));
    const std::vector<Row> rows = {{1, "one", "p0"}, {2, "two", "p0"}};
    for (int64_t identifier = 0; identifier < 2; ++identifier) {
        const std::vector<Row> expected_rows = identifier == 0 ? rows : std::vector<Row>{};
        const std::vector<RecordBatch::RowKind> kinds =
            identifier == 1 ? std::vector<RecordBatch::RowKind>(2, RecordBatch::RowKind::DELETE)
                            : std::vector<RecordBatch::RowKind>{};
        ASSERT_OK_AND_ASSIGN(
            std::unique_ptr<RecordBatch> batch,
            MakeBatch(rows, /*partitioned=*/false, /*bucket=*/0, /*row_kinds=*/kinds));
        ASSERT_OK(writer->Write(std::move(batch)));
        // Check both the initial rows and their deletion before either is committed.
        ASSERT_OK_AND_ASSIGN(std::vector<Row> realtime_rows, ReadRows(realtime_context));
        ASSERT_EQ(expected_rows, realtime_rows);
        ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> progress,
                             writer->PrepareCommitWithProgress(/*commit_identifier=*/identifier));
        ASSERT_OK_AND_ASSIGN(int64_t snapshot, Commit(progress, /*commit_identifier=*/identifier));
        ASSERT_OK_AND_ASSIGN(std::vector<Row> disk_rows, ReadRows());
        ASSERT_EQ(expected_rows, disk_rows);
        ASSERT_OK(writer->RefreshCommittedSnapshot(snapshot));
        ASSERT_OK_AND_ASSIGN(realtime_rows, ReadRows(realtime_context));
        ASSERT_EQ(expected_rows, realtime_rows);
    }
    ASSERT_OK_AND_ASSIGN(RealtimeOffsetMap offsets_before, ReadCommittedOffsets());
    ASSERT_OK_AND_ASSIGN(Snapshot compacted,
                         CompactAndCommit(/*partition=*/{}, /*bucket=*/0, /*commit_identifier=*/2));
    ASSERT_EQ(3, compacted.Id());
    ASSERT_EQ(Snapshot::CommitKind::Compact(), compacted.GetCommitKind());
    ASSERT_OK_AND_ASSIGN(std::shared_ptr<Plan> plan,
                         CreatePlan(/*realtime_context=*/nullptr, /*predicate=*/nullptr));
    ASSERT_TRUE(plan->Splits().empty());
    // Empty split metadata alone is not enough: read through the disk and realtime paths.
    ASSERT_OK_AND_ASSIGN(std::vector<Row> disk_rows, ReadRows(plan, nullptr));
    ASSERT_TRUE(disk_rows.empty());
    ASSERT_OK_AND_ASSIGN(std::vector<Row> realtime_rows, ReadRows(realtime_context));
    ASSERT_TRUE(realtime_rows.empty());
    ASSERT_OK_AND_ASSIGN(RealtimeOffsetMap offsets_after, ReadCommittedOffsets());
    ASSERT_EQ(offsets_before, offsets_after);
    ASSERT_OK(writer->RefreshCommittedSnapshot(compacted.Id()));
    ASSERT_OK_AND_ASSIGN(realtime_rows, ReadRows(realtime_context));
    ASSERT_TRUE(realtime_rows.empty());
    ASSERT_OK(writer->Close());
}

TEST_F(RealtimeWriteInteTest, TestDeletesAllRowsAfterCompactionRewrite) {
    ConfigureRealtimeDeduplicateTable();
    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
                         RealtimeContext::Create());
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
                         CreateRealtimeWriter(realtime_context));
    const std::vector<Row> rows = {{1, "one", "p0"}, {2, "two", "p0"}, {3, "three", "p0"}};
    ASSERT_OK(WriteCommitAndRefresh(rows, /*partitioned=*/false, /*bucket=*/0, /*row_kinds=*/{},
                                    /*commit_identifier=*/0, writer.get()));
    std::unique_ptr<FileStoreWrite> compactor;
    ASSERT_OK_AND_ASSIGN(
        std::vector<std::shared_ptr<CommitMessage>> messages,
        CompactAndPrepareCommit({RealtimePartitionBucket(/*partition=*/{}, /*bucket=*/0)},
                                /*commit_identifier=*/0, /*full_compaction=*/true, &compactor));
    ASSERT_EQ(1, messages.size());
    auto message = std::dynamic_pointer_cast<CommitMessageImpl>(messages[0]);
    ASSERT_NE(nullptr, message);
    ASSERT_EQ(1, message->GetCompactIncrement().CompactAfter().size());
    const auto& output = message->GetCompactIncrement().CompactAfter()[0];
    ASSERT_EQ(3, output->row_count);

    // D0 is empty; D1 deletes every row only after a real output has been written.
    ASSERT_OK(WriteCommitAndRefresh(
        rows, /*partitioned=*/false, /*bucket=*/0,
        /*row_kinds=*/std::vector<RecordBatch::RowKind>(3, RecordBatch::RowKind::DELETE),
        /*commit_identifier=*/1, writer.get()));
    {
        ASSERT_OK_AND_ASSIGN(std::vector<Row> disk_rows, ReadRows());
        ASSERT_TRUE(disk_rows.empty());
        ASSERT_OK_AND_ASSIGN(std::vector<Row> realtime_rows, ReadRows(realtime_context));
        ASSERT_TRUE(realtime_rows.empty());
    }
    ASSERT_OK_AND_ASSIGN(RealtimeOffsetMap offsets, ReadCommittedOffsets());
    ASSERT_EQ(1, offsets.size());
    ASSERT_EQ(6, offsets.at(RealtimePartitionBucket(/*partition=*/{}, /*bucket=*/0)));
    ASSERT_OK_AND_ASSIGN(Snapshot compacted, CommitCompaction(messages, /*commit_identifier=*/0));
    ASSERT_EQ(3, compacted.Id());
    ASSERT_OK_AND_ASSIGN(std::vector<FileState> files, InspectFiles());
    ASSERT_EQ(1, files.size());
    ASSERT_EQ(output->file_name, files[0].file->file_name);
    ASSERT_EQ(3, files[0].file->row_count);
    ASSERT_TRUE(files[0].deletion_file);
    ASSERT_EQ((std::set<int64_t>{0, 1, 2}), files[0].deleted_positions);
    {
        ASSERT_OK_AND_ASSIGN(std::vector<Row> disk_rows, ReadRows());
        ASSERT_TRUE(disk_rows.empty());
        ASSERT_OK_AND_ASSIGN(std::vector<Row> realtime_rows, ReadRows(realtime_context));
        ASSERT_TRUE(realtime_rows.empty());
    }
    ASSERT_OK(writer->RefreshCommittedSnapshot(compacted.Id()));
    {
        ASSERT_OK_AND_ASSIGN(std::vector<Row> disk_rows, ReadRows());
        ASSERT_TRUE(disk_rows.empty());
        ASSERT_OK_AND_ASSIGN(std::vector<Row> realtime_rows, ReadRows(realtime_context));
        ASSERT_TRUE(realtime_rows.empty());
    }
    ASSERT_OK_AND_ASSIGN(RealtimeOffsetMap after, ReadCommittedOffsets());
    ASSERT_EQ(offsets, after);
    ASSERT_OK(compactor->Close());

    // A later rewrite retires the fully deleted output.
    ASSERT_OK_AND_ASSIGN(
        Snapshot reclaimed,
        CompactCommitAndRefresh(/*partition=*/{}, /*bucket=*/0, /*commit_identifier=*/2,
                                /*full_compaction=*/true, writer.get()));
    ASSERT_EQ(4, reclaimed.Id());
    ASSERT_OK_AND_ASSIGN(files, InspectFiles());
    ASSERT_TRUE(files.empty());
    ASSERT_OK_AND_ASSIGN(after, ReadCommittedOffsets());
    ASSERT_EQ(offsets, after);
    {
        ASSERT_OK_AND_ASSIGN(std::vector<Row> disk_rows, ReadRows());
        ASSERT_TRUE(disk_rows.empty());
        ASSERT_OK_AND_ASSIGN(std::vector<Row> realtime_rows, ReadRows(realtime_context));
        ASSERT_TRUE(realtime_rows.empty());
    }
    ASSERT_OK(writer->Close());
}

TEST_F(RealtimeWriteInteTest, TestSparseDeletedOffsetsAcrossReaderBatches) {
    options_[Options::READ_BATCH_SIZE] = "3";
    // Keep holes in the raw batch so NextBatchWithBitmap must honor its sparse selection.
    options_["parquet.read.enable-page-index-filter"] = "false";
    ConfigureRealtimeDeduplicateTable();
    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
                         RealtimeContext::Create());
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
                         CreateRealtimeWriter(realtime_context));
    ASSERT_OK(WriteCommitAndRefresh(MakeRows(/*first_id=*/0, /*count=*/9, /*partition=*/"p0"),
                                    /*partitioned=*/false, /*bucket=*/0,
                                    /*row_kinds=*/{}, /*commit_identifier=*/0, writer.get()));
    ASSERT_OK(WriteCommitAndRefresh(MakeRows(/*first_id=*/9, /*count=*/9, /*partition=*/"p0"),
                                    /*partitioned=*/false, /*bucket=*/0,
                                    /*row_kinds=*/{}, /*commit_identifier=*/1, writer.get()));
    std::unique_ptr<FileStoreWrite> compactor;
    ASSERT_OK_AND_ASSIGN(
        std::vector<std::shared_ptr<CommitMessage>> messages,
        CompactAndPrepareCommit({RealtimePartitionBucket(/*partition=*/{}, /*bucket=*/0)},
                                /*commit_identifier=*/0, /*full_compaction=*/true, &compactor));
    ASSERT_EQ(1, messages.size());
    auto message = std::dynamic_pointer_cast<CommitMessageImpl>(messages[0]);
    ASSERT_NE(nullptr, message);
    ASSERT_EQ(2, message->GetCompactIncrement().CompactBefore().size());
    const std::vector<Row> deleted = {{0, "ignored", "p0"},  {2, "ignored", "p0"},
                                      {7, "ignored", "p0"},  {10, "ignored", "p0"},
                                      {14, "ignored", "p0"}, {17, "ignored", "p0"}};
    ASSERT_OK(WriteCommitAndRefresh(
        deleted, /*partitioned=*/false, /*bucket=*/0, /*row_kinds=*/
        std::vector<RecordBatch::RowKind>(deleted.size(), RecordBatch::RowKind::DELETE),
        /*commit_identifier=*/2, writer.get()));
    std::vector<Row> expected;
    const std::set<int64_t> positions = {0, 2, 7, 10, 14, 17};
    for (const Row& row : MakeRows(/*first_id=*/0, /*count=*/18, /*partition=*/"p0")) {
        if (positions.count(std::get<0>(row)) == 0) {
            expected.push_back(row);
        }
    }
    const std::multiset<Row> expected_rows(expected.begin(), expected.end());
    {
        ASSERT_OK_AND_ASSIGN(std::vector<Row> disk_rows, ReadRows());
        ASSERT_EQ(expected_rows, std::multiset<Row>(disk_rows.begin(), disk_rows.end()));
        ASSERT_OK_AND_ASSIGN(std::vector<Row> realtime_rows, ReadRows(realtime_context));
        ASSERT_EQ(expected_rows, std::multiset<Row>(realtime_rows.begin(), realtime_rows.end()));
    }
    ASSERT_OK_AND_ASSIGN(Snapshot compacted, CommitCompaction(messages, /*commit_identifier=*/0));
    ASSERT_OK_AND_ASSIGN(std::vector<FileState> files, InspectFiles());
    ASSERT_EQ(1, files.size());
    ASSERT_EQ(18, files[0].file->row_count);
    ASSERT_EQ(positions, files[0].deleted_positions);
    {
        ASSERT_OK_AND_ASSIGN(std::vector<Row> disk_rows, ReadRows());
        ASSERT_EQ(expected_rows, std::multiset<Row>(disk_rows.begin(), disk_rows.end()));
        ASSERT_OK_AND_ASSIGN(std::vector<Row> realtime_rows, ReadRows(realtime_context));
        ASSERT_EQ(expected_rows, std::multiset<Row>(realtime_rows.begin(), realtime_rows.end()));
    }
    ASSERT_OK(writer->RefreshCommittedSnapshot(compacted.Id()));
    {
        ASSERT_OK_AND_ASSIGN(std::vector<Row> disk_rows, ReadRows());
        ASSERT_EQ(expected_rows, std::multiset<Row>(disk_rows.begin(), disk_rows.end()));
        ASSERT_OK_AND_ASSIGN(std::vector<Row> realtime_rows, ReadRows(realtime_context));
        ASSERT_EQ(expected_rows, std::multiset<Row>(realtime_rows.begin(), realtime_rows.end()));
    }
    ASSERT_OK(writer->Close());
    ASSERT_OK(compactor->Close());
}

TEST_F(RealtimeWriteInteTest, TestCompactionRollsAndRemapsMultipleOutputs) {
    ConfigureRealtimeDeduplicateTable();
    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
                         RealtimeContext::Create());
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
                         CreateRealtimeWriter(realtime_context));
    ASSERT_OK(WriteCommitAndRefresh(MakeRows(/*first_id=*/0, /*count=*/1200, /*partition=*/"p0"),
                                    /*partitioned=*/false, /*bucket=*/0,
                                    /*row_kinds=*/{}, /*commit_identifier=*/0, writer.get()));
    ASSERT_OK(WriteCommitAndRefresh(MakeRows(/*first_id=*/1200, /*count=*/1200, /*partition=*/"p0"),
                                    /*partitioned=*/false, /*bucket=*/0,
                                    /*row_kinds=*/{}, /*commit_identifier=*/1, writer.get()));
    // Parquet checks target size every 1000 records, independently of target-file-row-num.
    options_[Options::TARGET_FILE_SIZE] = "1";
    options_[Options::READ_BATCH_SIZE] = "100";
    options_[Options::COMPACTION_FORCE_REWRITE_ALL_FILES] = "true";
    std::unique_ptr<FileStoreWrite> compactor;
    ASSERT_OK_AND_ASSIGN(
        std::vector<std::shared_ptr<CommitMessage>> messages,
        CompactAndPrepareCommit({RealtimePartitionBucket(/*partition=*/{}, /*bucket=*/0)},
                                /*commit_identifier=*/0, /*full_compaction=*/true, &compactor));
    ASSERT_EQ(1, messages.size());
    auto message = std::dynamic_pointer_cast<CommitMessageImpl>(messages[0]);
    ASSERT_NE(nullptr, message);
    const auto& outputs = message->GetCompactIncrement().CompactAfter();
    ASSERT_GT(outputs.size(), 1);
    // Pick one row from every physical output, including across the original file boundary.
    std::vector<Row> deleted;
    std::map<std::string, std::set<int64_t>> expected_dvs;
    int64_t first_id = 0;
    for (const auto& output : outputs) {
        ASSERT_GT(output->row_count, 1);
        deleted.emplace_back(first_id, "ignored", "p0");
        expected_dvs[output->file_name] = {0};
        first_id += output->row_count;
        bool has_offset_sidecar = false;
        for (const auto& extra : output->extra_files) {
            ASSERT_TRUE(extra);
            has_offset_sidecar |=
                extra->size() >= 7 && extra->compare(extra->size() - 7, 7, ".offset") == 0;
            ASSERT_OK_AND_ASSIGN(bool exists,
                                 dir_->GetFileSystem()->Exists(PathUtil::JoinPath(
                                     PathUtil::JoinPath(table_path_, "bucket-0"), *extra)));
            ASSERT_TRUE(exists);
        }
        ASSERT_TRUE(has_offset_sidecar);
    }
    ASSERT_EQ(2400, first_id);
    ASSERT_OK(WriteCommitAndRefresh(
        deleted, /*partitioned=*/false, /*bucket=*/0, /*row_kinds=*/
        std::vector<RecordBatch::RowKind>(deleted.size(), RecordBatch::RowKind::DELETE),
        /*commit_identifier=*/2, writer.get()));
    std::vector<Row> expected = MakeRows(/*first_id=*/0, /*count=*/2400, /*partition=*/"p0");
    expected.erase(std::remove_if(expected.begin(), expected.end(),
                                  [&](const Row& row) {
                                      return std::any_of(
                                          deleted.begin(), deleted.end(), [&](const Row& deletion) {
                                              return std::get<0>(row) == std::get<0>(deletion);
                                          });
                                  }),
                   expected.end());
    ASSERT_OK_AND_ASSIGN(Snapshot compacted, CommitCompaction(messages, /*commit_identifier=*/0));
    ASSERT_OK_AND_ASSIGN(std::vector<FileState> files, InspectFiles());
    ASSERT_EQ(outputs.size(), files.size());
    for (const FileState& file : files) {
        ASSERT_EQ(1, expected_dvs.count(file.file->file_name));
        ASSERT_EQ(expected_dvs.at(file.file->file_name), file.deleted_positions);
    }
    std::multiset<Row> expected_rows(expected.begin(), expected.end());
    {
        ASSERT_OK_AND_ASSIGN(std::vector<Row> disk_rows, ReadRows());
        ASSERT_EQ(expected_rows, std::multiset<Row>(disk_rows.begin(), disk_rows.end()));
        ASSERT_OK_AND_ASSIGN(std::vector<Row> realtime_rows, ReadRows(realtime_context));
        ASSERT_EQ(expected_rows, std::multiset<Row>(realtime_rows.begin(), realtime_rows.end()));
    }
    ASSERT_OK(writer->RefreshCommittedSnapshot(compacted.Id()));
    {
        ASSERT_OK_AND_ASSIGN(std::vector<Row> disk_rows, ReadRows());
        ASSERT_EQ(expected_rows, std::multiset<Row>(disk_rows.begin(), disk_rows.end()));
        ASSERT_OK_AND_ASSIGN(std::vector<Row> realtime_rows, ReadRows(realtime_context));
        ASSERT_EQ(expected_rows, std::multiset<Row>(realtime_rows.begin(), realtime_rows.end()));
    }
    ASSERT_OK(writer->Close());
    ASSERT_OK(compactor->Close());

    // Restore every rolled key->offset sidecar and delete a surviving row from every output.
    ASSERT_OK_AND_ASSIGN(realtime_context, RealtimeContext::Create());
    ASSERT_OK_AND_ASSIGN(writer, CreateRealtimeWriter(realtime_context));
    {
        ASSERT_OK_AND_ASSIGN(std::vector<Row> disk_rows, ReadRows());
        ASSERT_EQ(expected_rows, std::multiset<Row>(disk_rows.begin(), disk_rows.end()));
        ASSERT_OK_AND_ASSIGN(std::vector<Row> realtime_rows, ReadRows(realtime_context));
        ASSERT_EQ(expected_rows, std::multiset<Row>(realtime_rows.begin(), realtime_rows.end()));
    }
    std::vector<Row> restart_deletes;
    for (const Row& row : deleted) {
        restart_deletes.emplace_back(std::get<0>(row) + 1, "ignored", "p0");
    }
    ASSERT_OK(WriteCommitAndRefresh(
        restart_deletes, /*partitioned=*/false, /*bucket=*/0, /*row_kinds=*/
        std::vector<RecordBatch::RowKind>(restart_deletes.size(), RecordBatch::RowKind::DELETE),
        /*commit_identifier=*/3, writer.get()));
    expected.erase(std::remove_if(expected.begin(), expected.end(),
                                  [&](const Row& row) {
                                      return std::any_of(
                                          restart_deletes.begin(), restart_deletes.end(),
                                          [&](const Row& deletion) {
                                              return std::get<0>(row) == std::get<0>(deletion);
                                          });
                                  }),
                   expected.end());
    expected_rows = std::multiset<Row>(expected.begin(), expected.end());
    {
        ASSERT_OK_AND_ASSIGN(std::vector<Row> disk_rows, ReadRows());
        ASSERT_EQ(expected_rows, std::multiset<Row>(disk_rows.begin(), disk_rows.end()));
        ASSERT_OK_AND_ASSIGN(std::vector<Row> realtime_rows, ReadRows(realtime_context));
        ASSERT_EQ(expected_rows, std::multiset<Row>(realtime_rows.begin(), realtime_rows.end()));
    }
    ASSERT_OK_AND_ASSIGN(files, InspectFiles());
    ASSERT_EQ(outputs.size(), files.size());
    for (const FileState& file : files) {
        ASSERT_EQ(1, expected_dvs.count(file.file->file_name));
        ASSERT_EQ((std::set<int64_t>{0, 1}), file.deleted_positions);
    }
    ASSERT_OK(writer->Close());
}

TEST_F(RealtimeWriteInteTest, TestCompactionIsolatesPartitionsAndBuckets) {
    options_[Options::BUCKET] = "2";
    ConfigureRealtimeDeduplicateTable(/*partition_keys=*/{"pt"});
    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
                         RealtimeContext::Create());
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
                         CreateRealtimeWriter(realtime_context));
    const std::vector<std::pair<std::string, int32_t>> scopes = {
        {"p0", 0}, {"p0", 1}, {"p1", 0}, {"p1", 1}};
    for (const auto& [partition, bucket] : scopes) {
        const std::vector<Row> rows = {
            {0, fmt::format("{}-b{}-zero", partition, bucket), partition},
            {1, fmt::format("{}-b{}-one", partition, bucket), partition},
            {2, fmt::format("{}-b{}-two", partition, bucket), partition}};
        ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> batch,
                             MakeBatch(rows, /*partitioned=*/true, /*bucket=*/bucket));
        ASSERT_OK(writer->Write(std::move(batch)));
    }
    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> initial,
                         writer->PrepareCommitWithProgress(/*commit_identifier=*/0));
    ASSERT_EQ(4, initial.size());
    ASSERT_OK_AND_ASSIGN(int64_t snapshot, Commit(initial, /*commit_identifier=*/0));
    ASSERT_OK(writer->RefreshCommittedSnapshot(snapshot));
    // The fourth scope is not compacted. Its existing DV must survive the other replacements.
    ASSERT_OK(WriteCommitAndRefresh({{0, "ignored", "p1"}}, /*partitioned=*/true, /*bucket=*/1,
                                    /*row_kinds=*/{RecordBatch::RowKind::DELETE},
                                    /*commit_identifier=*/1, writer.get()));
    ASSERT_OK_AND_ASSIGN(std::vector<FileState> baseline, InspectFiles());
    std::optional<DeletionFile> untouched_dv;
    std::string untouched_name;
    for (const FileState& file : baseline) {
        if (file.deletion_file) {
            ASSERT_FALSE(untouched_dv);
            untouched_dv = file.deletion_file;
            untouched_name = file.file->file_name;
        }
    }
    ASSERT_TRUE(untouched_dv);
    std::vector<RealtimePartitionBucket> compact_scopes;
    for (size_t i = 0; i < 3; ++i) {
        compact_scopes.emplace_back(std::map<std::string, std::string>{{"pt", scopes[i].first}},
                                    scopes[i].second);
    }
    std::unique_ptr<FileStoreWrite> compactor;
    ASSERT_OK_AND_ASSIGN(std::vector<std::shared_ptr<CommitMessage>> messages,
                         CompactAndPrepareCommit(compact_scopes, /*commit_identifier=*/0,
                                                 /*full_compaction=*/true, &compactor));
    ASSERT_EQ(3, messages.size());
    std::map<std::string, std::set<int64_t>> expected_dvs;
    for (const auto& raw_message : messages) {
        auto message = std::dynamic_pointer_cast<CommitMessageImpl>(raw_message);
        ASSERT_NE(nullptr, message);
        ASSERT_EQ(1, message->GetCompactIncrement().CompactAfter().size());
        const std::string& name = message->GetCompactIncrement().CompactAfter()[0]->file_name;
        // Resolve the scope through the bucket path of the corresponding compact-before file.
        ASSERT_EQ(1, message->GetCompactIncrement().CompactBefore().size());
        const std::string& before = message->GetCompactIncrement().CompactBefore()[0]->file_name;
        bool found = false;
        for (const FileState& file : baseline) {
            if (file.file->file_name != before) {
                continue;
            }
            for (size_t i = 0; i < 3; ++i) {
                const std::string suffix =
                    fmt::format("pt={}/bucket-{}", scopes[i].first, scopes[i].second);
                if (file.bucket_path == PathUtil::JoinPath(table_path_, suffix)) {
                    expected_dvs[name] = {static_cast<int64_t>(i)};
                    found = true;
                }
            }
        }
        ASSERT_TRUE(found);
    }
    // Every scope starts with offsets {0,1,2}; different deletions expose cross-scope leakage.
    for (size_t i = 0; i < 3; ++i) {
        ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> batch,
                             MakeBatch({{static_cast<int64_t>(i), "ignored", scopes[i].first}},
                                       /*partitioned=*/true, /*bucket=*/scopes[i].second,
                                       /*row_kinds=*/{RecordBatch::RowKind::DELETE}));
        ASSERT_OK(writer->Write(std::move(batch)));
    }
    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> deleted,
                         writer->PrepareCommitWithProgress(/*commit_identifier=*/2));
    ASSERT_EQ(3, deleted.size());
    ASSERT_OK_AND_ASSIGN(snapshot, Commit(deleted, /*commit_identifier=*/2));
    ASSERT_OK(writer->RefreshCommittedSnapshot(snapshot));
    ASSERT_OK_AND_ASSIGN(RealtimeOffsetMap offsets, ReadCommittedOffsets());
    ASSERT_EQ(4, offsets.size());
    for (const auto& [partition, bucket] : scopes) {
        ASSERT_EQ(4, offsets.at(RealtimePartitionBucket(/*partition=*/{{"pt", partition}},
                                                        /*bucket=*/bucket)));
    }
    const std::multiset<Row> expected_rows = {
        // p0/b0: id 0 was deleted.
        {1, "p0-b0-one", "p0"},
        {2, "p0-b0-two", "p0"},
        // p0/b1: id 1 was deleted.
        {0, "p0-b1-zero", "p0"},
        {2, "p0-b1-two", "p0"},
        // p1/b0: id 2 was deleted.
        {0, "p1-b0-zero", "p1"},
        {1, "p1-b0-one", "p1"},
        // p1/b1: id 0 was already deleted; this bucket is not compacted.
        {1, "p1-b1-one", "p1"},
        {2, "p1-b1-two", "p1"},
    };
    {
        ASSERT_OK_AND_ASSIGN(std::vector<Row> disk_rows, ReadRows());
        ASSERT_EQ(expected_rows, std::multiset<Row>(disk_rows.begin(), disk_rows.end()));
        ASSERT_OK_AND_ASSIGN(std::vector<Row> realtime_rows, ReadRows(realtime_context));
        ASSERT_EQ(expected_rows, std::multiset<Row>(realtime_rows.begin(), realtime_rows.end()));
    }
    ASSERT_OK_AND_ASSIGN(Snapshot compacted, CommitCompaction(messages, /*commit_identifier=*/0));
    ASSERT_EQ(4, compacted.Id());
    ASSERT_OK_AND_ASSIGN(std::vector<FileState> files, InspectFiles());
    ASSERT_EQ(4, files.size());
    size_t checked_outputs = 0;
    bool checked_untouched = false;
    for (const FileState& file : files) {
        if (file.file->file_name == untouched_name) {
            checked_untouched = true;
            ASSERT_TRUE(file.deletion_file);
            ASSERT_EQ(untouched_dv->path, file.deletion_file->path);
            ASSERT_EQ(untouched_dv->offset, file.deletion_file->offset);
            ASSERT_EQ((std::set<int64_t>{0}), file.deleted_positions);
        } else {
            ASSERT_EQ(1, expected_dvs.count(file.file->file_name));
            ASSERT_EQ(expected_dvs.at(file.file->file_name), file.deleted_positions);
            ++checked_outputs;
        }
    }
    ASSERT_TRUE(checked_untouched);
    ASSERT_EQ(3, checked_outputs);
    ASSERT_OK_AND_ASSIGN(RealtimeOffsetMap after, ReadCommittedOffsets());
    ASSERT_EQ(offsets, after);
    {
        ASSERT_OK_AND_ASSIGN(std::vector<Row> disk_rows, ReadRows());
        ASSERT_EQ(expected_rows, std::multiset<Row>(disk_rows.begin(), disk_rows.end()));
        ASSERT_OK_AND_ASSIGN(std::vector<Row> realtime_rows, ReadRows(realtime_context));
        ASSERT_EQ(expected_rows, std::multiset<Row>(realtime_rows.begin(), realtime_rows.end()));
    }
    ASSERT_OK(writer->RefreshCommittedSnapshot(compacted.Id()));
    {
        ASSERT_OK_AND_ASSIGN(std::vector<Row> disk_rows, ReadRows());
        ASSERT_EQ(expected_rows, std::multiset<Row>(disk_rows.begin(), disk_rows.end()));
        ASSERT_OK_AND_ASSIGN(std::vector<Row> realtime_rows, ReadRows(realtime_context));
        ASSERT_EQ(expected_rows, std::multiset<Row>(realtime_rows.begin(), realtime_rows.end()));
    }
    ASSERT_OK(writer->Close());
    ASSERT_OK(compactor->Close());
}

TEST_F(RealtimeWriteInteTest, TestNoHitPreservesBaselineDv) {
    ConfigureRealtimeDeduplicateTable();
    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
                         RealtimeContext::Create());
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
                         CreateRealtimeWriter(realtime_context));
    ASSERT_OK(WriteCommitAndRefresh({{0, "zero", "p0"}, {1, "one", "p0"}, {2, "two", "p0"}},
                                    /*partitioned=*/false, /*bucket=*/0, /*row_kinds=*/{},
                                    /*commit_identifier=*/0, writer.get()));
    ASSERT_OK(WriteCommitAndRefresh({{0, "ignored", "p0"}}, /*partitioned=*/false, /*bucket=*/0,
                                    /*row_kinds=*/{RecordBatch::RowKind::DELETE},
                                    /*commit_identifier=*/1, writer.get()));
    ASSERT_OK(CompactCommitAndRefresh(/*partition=*/{}, /*bucket=*/0, /*commit_identifier=*/2,
                                      /*full_compaction=*/true, writer.get()));
    // Offset 0 is now physically absent, while a different offset creates a nonempty baseline.
    ASSERT_OK(WriteCommitAndRefresh({{1, "new-one", "p0"}}, /*partitioned=*/false, /*bucket=*/0,
                                    /*row_kinds=*/{RecordBatch::RowKind::UPDATE_AFTER},
                                    /*commit_identifier=*/3, writer.get()));
    ASSERT_OK_AND_ASSIGN(std::vector<FileState> baseline, InspectFiles());
    SnapshotManager snapshot_manager(dir_->GetFileSystem(), table_path_);
    ASSERT_OK_AND_ASSIGN(std::optional<Snapshot> before, snapshot_manager.LatestSnapshot());
    ASSERT_TRUE(before);
    ASSERT_TRUE(before->IndexManifest());
    std::set<std::string> indexes_before;
    {
        std::vector<BasicFileStatus> index_files;
        ASSERT_OK(
            dir_->GetFileSystem()->ListDir(PathUtil::JoinPath(table_path_, "index"), &index_files));
        for (const auto& file : index_files) {
            if (!file.IsDir()) {
                indexes_before.insert(file.GetPath());
            }
        }
    }
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> batch,
                         MakeBatch({{3, "three", "p0"}}, /*partitioned=*/false));
    ASSERT_OK(writer->Write(std::move(batch)));
    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> progress,
                         writer->PrepareCommitWithProgress(/*commit_identifier=*/4));
    ASSERT_EQ(1, progress.size());
    // Replay a deletion whose offset has already been removed by compaction.
    auto no_hit = std::make_shared<RoaringBitmap64>();
    no_hit->Add(0);
    progress[0].deleted_offsets = no_hit;
    CommitContextBuilder builder(table_path_, commit_user_);
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<CommitContext> context,
                         builder.SetOptions(options_).Finish());
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreCommit> commit,
                         FileStoreCommit::Create(std::move(context)));
    ASSERT_OK_AND_ASSIGN(
        int64_t snapshot,
        commit->CommitWithProgress(progress, /*commit_identifier=*/4, /*watermark=*/std::nullopt));
    ASSERT_EQ(5, snapshot);
    ASSERT_OK_AND_ASSIGN(std::optional<Snapshot> after, snapshot_manager.LatestSnapshot());
    ASSERT_TRUE(after);
    ASSERT_EQ(before->IndexManifest(), after->IndexManifest());
    std::set<std::string> indexes_after;
    {
        std::vector<BasicFileStatus> index_files;
        ASSERT_OK(
            dir_->GetFileSystem()->ListDir(PathUtil::JoinPath(table_path_, "index"), &index_files));
        for (const auto& file : index_files) {
            if (!file.IsDir()) {
                indexes_after.insert(file.GetPath());
            }
        }
    }
    ASSERT_EQ(indexes_before, indexes_after);
    ASSERT_OK_AND_ASSIGN(std::vector<FileState> files, InspectFiles());
    size_t checked = 0;
    for (const FileState& old : baseline) {
        if (!old.deletion_file) {
            continue;
        }
        for (const FileState& file : files) {
            if (old.file->file_name == file.file->file_name) {
                ASSERT_TRUE(file.deletion_file);
                ASSERT_EQ(old.deletion_file->path, file.deletion_file->path);
                ASSERT_EQ(old.deletion_file->offset, file.deletion_file->offset);
                ASSERT_EQ(old.deletion_file->length, file.deletion_file->length);
                ASSERT_EQ(old.deleted_positions, file.deleted_positions);
                ++checked;
            }
        }
    }
    ASSERT_EQ(1, checked);
    const std::vector<Row> expected = {{2, "two", "p0"}, {1, "new-one", "p0"}, {3, "three", "p0"}};
    const std::multiset<Row> expected_rows(expected.begin(), expected.end());
    {
        ASSERT_OK_AND_ASSIGN(std::vector<Row> disk_rows, ReadRows());
        ASSERT_EQ(expected_rows, std::multiset<Row>(disk_rows.begin(), disk_rows.end()));
        ASSERT_OK_AND_ASSIGN(std::vector<Row> realtime_rows, ReadRows(realtime_context));
        ASSERT_EQ(expected_rows, std::multiset<Row>(realtime_rows.begin(), realtime_rows.end()));
    }
    ASSERT_OK(writer->RefreshCommittedSnapshot(snapshot));
    {
        ASSERT_OK_AND_ASSIGN(std::vector<Row> disk_rows, ReadRows());
        ASSERT_EQ(expected_rows, std::multiset<Row>(disk_rows.begin(), disk_rows.end()));
        ASSERT_OK_AND_ASSIGN(std::vector<Row> realtime_rows, ReadRows(realtime_context));
        ASSERT_EQ(expected_rows, std::multiset<Row>(realtime_rows.begin(), realtime_rows.end()));
    }
    ASSERT_OK(writer->Close());
}

TEST_F(RealtimeWriteInteTest, TestPublicationConflictCleansOnlyAttemptDv) {
    ConfigureRealtimeDeduplicateTable();
    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
                         RealtimeContext::Create());
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
                         CreateRealtimeWriter(realtime_context));
    ASSERT_OK(WriteCommitAndRefresh(MakeRows(/*first_id=*/0, /*count=*/5, /*partition=*/"p0"),
                                    /*partitioned=*/false, /*bucket=*/0,
                                    /*row_kinds=*/{}, /*commit_identifier=*/0, writer.get()));
    ASSERT_OK(WriteCommitAndRefresh({{0, "ignored", "p0"}}, /*partitioned=*/false, /*bucket=*/0,
                                    /*row_kinds=*/{RecordBatch::RowKind::DELETE},
                                    /*commit_identifier=*/1, writer.get()));
    std::unique_ptr<FileStoreWrite> compactor;
    ASSERT_OK_AND_ASSIGN(
        std::vector<std::shared_ptr<CommitMessage>> messages,
        CompactAndPrepareCommit({RealtimePartitionBucket(/*partition=*/{}, /*bucket=*/0)},
                                /*commit_identifier=*/0, /*full_compaction=*/true, &compactor));
    ASSERT_EQ(1, messages.size());
    auto message = std::dynamic_pointer_cast<CommitMessageImpl>(messages[0]);
    ASSERT_NE(nullptr, message);
    ASSERT_EQ(1, message->GetCompactIncrement().CompactAfter().size());
    const auto& output = message->GetCompactIncrement().CompactAfter()[0];
    ASSERT_EQ(4, output->row_count);
    ASSERT_OK(WriteCommitAndRefresh({{1, "ignored", "p0"}}, /*partitioned=*/false, /*bucket=*/0,
                                    /*row_kinds=*/{RecordBatch::RowKind::DELETE},
                                    /*commit_identifier=*/2, writer.get()));
    ASSERT_OK_AND_ASSIGN(std::vector<FileState> baseline, InspectFiles());
    ASSERT_EQ(1, baseline.size());
    ASSERT_TRUE(baseline[0].deletion_file);
    std::set<std::string> indexes_before;
    {
        std::vector<BasicFileStatus> index_files;
        ASSERT_OK(
            dir_->GetFileSystem()->ListDir(PathUtil::JoinPath(table_path_, "index"), &index_files));
        for (const auto& file : index_files) {
            if (!file.IsDir()) {
                indexes_before.insert(file.GetPath());
            }
        }
    }
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> batch,
                         MakeBatch({{2, "ignored", "p0"}}, /*partitioned=*/false, /*bucket=*/0,
                                   /*row_kinds=*/{RecordBatch::RowKind::DELETE}));
    ASSERT_OK(writer->Write(std::move(batch)));
    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> concurrent,
                         writer->PrepareCommitWithProgress(/*commit_identifier=*/3));
    std::set<std::string> failed_attempt_paths;
    auto fs = std::make_shared<InterleavingSnapshotFileSystem>(
        PathUtil::JoinPath(table_path_, "snapshot/snapshot-4"), [&]() -> Status {
            // Capture candidates before the competing build writes any new DV of its own.
            std::vector<BasicFileStatus> index_files;
            PAIMON_RETURN_NOT_OK(dir_->GetFileSystem()->ListDir(
                PathUtil::JoinPath(table_path_, "index"), &index_files));
            std::set<std::string> during;
            for (const auto& file : index_files) {
                if (!file.IsDir()) {
                    during.insert(file.GetPath());
                }
            }
            std::set_difference(during.begin(), during.end(), indexes_before.begin(),
                                indexes_before.end(),
                                std::inserter(failed_attempt_paths, failed_attempt_paths.end()));
            PAIMON_ASSIGN_OR_RAISE(int64_t snapshot, Commit(concurrent, /*commit_identifier=*/3));
            if (snapshot != 4) {
                return Status::Invalid("expected the competing snapshot to occupy snapshot 4");
            }
            return Status::OK();
        });
    CommitContextBuilder builder(table_path_, "regression-compactor");
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<CommitContext> context,
                         builder.SetOptions(options_).WithFileSystem(fs).Finish());
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreCommit> commit,
                         FileStoreCommit::Create(std::move(context)));
    ASSERT_OK(commit->Commit(messages, /*commit_identifier=*/0));
    ASSERT_OK_AND_ASSIGN(uint64_t attempts, commit->GetCommitMetrics()->GetCounter(
                                                CommitMetrics::LAST_COMMIT_ATTEMPTS));
    ASSERT_EQ(2, attempts);
    ASSERT_EQ(1, failed_attempt_paths.size());
    for (const std::string& path : failed_attempt_paths) {
        ASSERT_OK_AND_ASSIGN(bool exists, dir_->GetFileSystem()->Exists(path));
        ASSERT_FALSE(exists) << path;
    }
    // Cleanup owns only unpublished DVs, not baseline DVs, source data, or compact outputs.
    for (const std::string& path : indexes_before) {
        ASSERT_OK_AND_ASSIGN(bool exists, dir_->GetFileSystem()->Exists(path));
        ASSERT_TRUE(exists) << path;
    }
    std::vector<std::shared_ptr<DataFileMeta>> retained = {baseline[0].file, output};
    for (const auto& file : retained) {
        const std::string bucket_path = PathUtil::JoinPath(table_path_, "bucket-0");
        ASSERT_OK_AND_ASSIGN(bool exists, dir_->GetFileSystem()->Exists(
                                              PathUtil::JoinPath(bucket_path, file->file_name)));
        ASSERT_TRUE(exists);
        for (const auto& extra : file->extra_files) {
            ASSERT_TRUE(extra);
            ASSERT_OK_AND_ASSIGN(
                exists, dir_->GetFileSystem()->Exists(PathUtil::JoinPath(bucket_path, *extra)));
            ASSERT_TRUE(exists);
        }
    }
    ASSERT_OK_AND_ASSIGN(std::vector<FileState> files, InspectFiles());
    ASSERT_EQ(1, files.size());
    ASSERT_EQ(output->file_name, files[0].file->file_name);
    ASSERT_TRUE(files[0].deletion_file);
    ASSERT_EQ(0, failed_attempt_paths.count(files[0].deletion_file->path));
    ASSERT_EQ((std::set<int64_t>{0, 1}), files[0].deleted_positions);
    ASSERT_OK_AND_ASSIGN(bool exists, dir_->GetFileSystem()->Exists(files[0].deletion_file->path));
    ASSERT_TRUE(exists);
    const std::vector<Row> expected = MakeRows(/*first_id=*/3, /*count=*/2, /*partition=*/"p0");
    const std::multiset<Row> expected_rows(expected.begin(), expected.end());
    {
        ASSERT_OK_AND_ASSIGN(std::vector<Row> disk_rows, ReadRows());
        ASSERT_EQ(expected_rows, std::multiset<Row>(disk_rows.begin(), disk_rows.end()));
        ASSERT_OK_AND_ASSIGN(std::vector<Row> realtime_rows, ReadRows(realtime_context));
        ASSERT_EQ(expected_rows, std::multiset<Row>(realtime_rows.begin(), realtime_rows.end()));
    }
    SnapshotManager snapshot_manager(dir_->GetFileSystem(), table_path_);
    ASSERT_OK_AND_ASSIGN(std::optional<Snapshot> published, snapshot_manager.LatestSnapshot());
    ASSERT_TRUE(published);
    ASSERT_EQ(5, published->Id());
    ASSERT_OK(writer->RefreshCommittedSnapshot(published->Id()));
    {
        ASSERT_OK_AND_ASSIGN(std::vector<Row> disk_rows, ReadRows());
        ASSERT_EQ(expected_rows, std::multiset<Row>(disk_rows.begin(), disk_rows.end()));
        ASSERT_OK_AND_ASSIGN(std::vector<Row> realtime_rows, ReadRows(realtime_context));
        ASSERT_EQ(expected_rows, std::multiset<Row>(realtime_rows.begin(), realtime_rows.end()));
    }
    ASSERT_OK(writer->Close());
    ASSERT_OK(compactor->Close());
}

TEST_F(RealtimeWriteInteTest, TestRealtimeDeduplicateWriterHandoffRestoresCommittedFiles) {
    ConfigureRealtimeDeduplicateTable();

    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
                         RealtimeContext::Create());
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> first_writer,
                         CreateRealtimeWriter(realtime_context));
    const std::vector<Row> initial_rows = {{1, "old-one", "p0"}, {2, "two", "p0"}};
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> initial_batch,
                         MakeBatch(initial_rows, /*partitioned=*/false));
    ASSERT_OK(first_writer->Write(std::move(initial_batch)));
    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> initial_progress,
                         first_writer->PrepareCommitWithProgress(/*commit_identifier=*/0));
    ASSERT_OK_AND_ASSIGN(int64_t snapshot_id, Commit(initial_progress, /*commit_identifier=*/0));
    ASSERT_EQ(1, snapshot_id);
    ASSERT_OK(first_writer->Close());
    first_writer.reset();

    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> second_writer,
                         CreateRealtimeWriter(realtime_context));
    ASSERT_OK_AND_ASSIGN(std::vector<Row> restored_rows, ReadRows(realtime_context));
    ASSERT_EQ(initial_rows, restored_rows);

    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> update_batch,
                         MakeBatch({{1, "new-one", "p0"}}, /*partitioned=*/false));
    ASSERT_OK(second_writer->Write(std::move(update_batch)));
    const std::vector<Row> expected_rows = {{2, "two", "p0"}, {1, "new-one", "p0"}};
    ASSERT_OK_AND_ASSIGN(std::vector<Row> updated_rows, ReadRows(realtime_context));
    ASSERT_EQ(expected_rows, updated_rows);

    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> update_progress,
                         second_writer->PrepareCommitWithProgress(/*commit_identifier=*/1));
    ASSERT_EQ(1, update_progress.size());
    std::shared_ptr<CommitMessageImpl> update_message =
        std::dynamic_pointer_cast<CommitMessageImpl>(update_progress[0].commit_message);
    ASSERT_NE(nullptr, update_message);
    ASSERT_TRUE(update_message->GetCompactIncrement().NewIndexFiles().empty());
    ASSERT_NE(nullptr, update_progress[0].deleted_offsets);
    ASSERT_FALSE(update_progress[0].deleted_offsets->IsEmpty());
    ASSERT_OK_AND_ASSIGN(std::vector<Row> rows_after_prepare, ReadRows(realtime_context));
    ASSERT_EQ(expected_rows, rows_after_prepare);
    ASSERT_OK(second_writer->Close());
}

TEST_F(RealtimeWriteInteTest, TestRealtimeDeduplicateWriterHandoffRetainsPreparedFiles) {
    ConfigureRealtimeDeduplicateTable();
    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
                         RealtimeContext::Create());
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> first_writer,
                         CreateRealtimeWriter(realtime_context));
    const std::vector<Row> initial_rows = {
        {1, "old-one", "p0"}, {2, "keep-two", "p0"}, {3, "delete-three", "p0"}};
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> initial_batch,
                         MakeBatch(initial_rows, /*partitioned=*/false));
    ASSERT_OK(first_writer->Write(std::move(initial_batch)));
    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> progress,
                         first_writer->PrepareCommitWithProgress(/*commit_identifier=*/0));
    ASSERT_EQ(1, progress.size());
    ASSERT_EQ(OffsetRange(0, 3), progress[0].offset_range);
    std::shared_ptr<CommitMessageImpl> initial_message =
        std::dynamic_pointer_cast<CommitMessageImpl>(progress[0].commit_message);
    ASSERT_NE(nullptr, initial_message);
    ASSERT_EQ(1, initial_message->GetNewFilesIncrement().NewFiles().size());
    ASSERT_TRUE(initial_message->GetCompactIncrement().NewIndexFiles().empty());
    ASSERT_OK(first_writer->Close());
    first_writer.reset();

    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> second_writer,
                         CreateRealtimeWriter(realtime_context));
    ASSERT_OK_AND_ASSIGN(std::vector<Row> restored_rows, ReadRows(realtime_context));
    ASSERT_EQ(initial_rows, restored_rows);
    ASSERT_OK_AND_ASSIGN(
        std::unique_ptr<RecordBatch> update_batch,
        MakeBatch({Row{1, "new-one", "p0"}, Row{3, "ignored", "p0"}, Row{4, "four", "p0"}},
                  /*partitioned=*/false, /*bucket=*/0,
                  {RecordBatch::RowKind::UPDATE_AFTER, RecordBatch::RowKind::DELETE,
                   RecordBatch::RowKind::INSERT}));
    ASSERT_OK(second_writer->Write(std::move(update_batch)));
    const std::vector<Row> expected_rows = {
        {2, "keep-two", "p0"}, {1, "new-one", "p0"}, {4, "four", "p0"}};
    ASSERT_OK_AND_ASSIGN(std::vector<Row> rows_before_prepare, ReadRows(realtime_context));
    ASSERT_EQ(expected_rows, rows_before_prepare);

    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> update_progress,
                         second_writer->PrepareCommitWithProgress(/*commit_identifier=*/1));
    ASSERT_EQ(1, update_progress.size());
    ASSERT_EQ(OffsetRange(3, 6), update_progress[0].offset_range);
    std::shared_ptr<CommitMessageImpl> update_message =
        std::dynamic_pointer_cast<CommitMessageImpl>(update_progress[0].commit_message);
    ASSERT_NE(nullptr, update_message);
    ASSERT_EQ(1, update_message->GetNewFilesIncrement().NewFiles().size());
    ASSERT_EQ(2, update_message->GetNewFilesIncrement().NewFiles()[0]->row_count);
    ASSERT_TRUE(update_message->GetCompactIncrement().NewIndexFiles().empty());
    ASSERT_NE(nullptr, update_progress[0].deleted_offsets);
    ASSERT_FALSE(update_progress[0].deleted_offsets->IsEmpty());
    ASSERT_OK_AND_ASSIGN(std::vector<Row> rows_after_prepare, ReadRows(realtime_context));
    ASSERT_EQ(expected_rows, rows_after_prepare);

    progress.push_back(std::move(update_progress[0]));
    ASSERT_OK_AND_ASSIGN(int64_t snapshot_id, Commit(progress, /*commit_identifier=*/1));
    ASSERT_OK_AND_ASSIGN(std::vector<Row> rows_before_refresh, ReadRows(realtime_context));
    ASSERT_EQ(expected_rows, rows_before_refresh);
    ASSERT_OK_AND_ASSIGN(std::vector<Row> disk_rows, ReadRows());
    ASSERT_EQ(expected_rows, disk_rows);
    ASSERT_OK(second_writer->RefreshCommittedSnapshot(snapshot_id));
    ASSERT_OK_AND_ASSIGN(std::vector<Row> rows_after_refresh, ReadRows(realtime_context));
    ASSERT_EQ(expected_rows, rows_after_refresh);
    ASSERT_OK(second_writer->Close());
}

TEST_F(RealtimeWriteInteTest, TestRealtimeDeduplicateWriterHandoffRetainsDvAcrossRefresh) {
    for (bool prepare_before_refresh : {false, true}) {
        SCOPED_TRACE(prepare_before_refresh);
        dir_ = UniqueTestDirectory::Create("local");
        ASSERT_NE(nullptr, dir_);
        table_path_ = PathUtil::JoinPath(dir_->Str(), "foo.db/bar");
        ResetExternalOffset(/*partition=*/{}, /*bucket=*/0, /*next_offset=*/0);
        ConfigureRealtimeDeduplicateTable();
        ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
                             RealtimeContext::Create());
        ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> first_writer,
                             CreateRealtimeWriter(realtime_context));
        const std::vector<Row> initial_rows = {
            {1, "one", "p0"}, {2, "two", "p0"}, {3, "three", "p0"}};
        ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> initial_batch,
                             MakeBatch(initial_rows, /*partitioned=*/false));
        ASSERT_OK(first_writer->Write(std::move(initial_batch)));
        ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> initial_progress,
                             first_writer->PrepareCommitWithProgress(/*commit_identifier=*/0));
        ASSERT_EQ(1, initial_progress.size());
        auto initial_message =
            std::dynamic_pointer_cast<CommitMessageImpl>(initial_progress[0].commit_message);
        ASSERT_NE(nullptr, initial_message);
        ASSERT_EQ(1, initial_message->GetNewFilesIncrement().NewFiles().size());
        ASSERT_OK_AND_ASSIGN(int64_t initial_snapshot_id,
                             Commit(initial_progress, /*commit_identifier=*/0));
        ASSERT_OK(first_writer->RefreshCommittedSnapshot(initial_snapshot_id));
        ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> first_delete,
                             MakeBatch({initial_rows[0]}, /*partitioned=*/false, /*bucket=*/0,
                                       {RecordBatch::RowKind::DELETE}));
        ASSERT_OK(first_writer->Write(std::move(first_delete)));
        ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> first_progress,
                             first_writer->PrepareCommitWithProgress(/*commit_identifier=*/1));
        ASSERT_OK(first_writer->Close());
        first_writer.reset();

        ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> second_writer,
                             CreateRealtimeWriter(realtime_context));
        const Row fourth_row = {4, "four", "p0"};
        ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> insert_batch,
                             MakeBatch({fourth_row}, /*partitioned=*/false));
        // Create the bucket writer before the first writer's deletion reaches the snapshot.
        ASSERT_OK(second_writer->Write(std::move(insert_batch)));
        std::vector<RealtimeCommitProgress> second_progress;
        if (prepare_before_refresh) {
            ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> second_delete,
                                 MakeBatch({initial_rows[1]}, /*partitioned=*/false, /*bucket=*/0,
                                           {RecordBatch::RowKind::DELETE}));
            ASSERT_OK(second_writer->Write(std::move(second_delete)));
            ASSERT_OK_AND_ASSIGN(second_progress,
                                 second_writer->PrepareCommitWithProgress(/*commit_identifier=*/2));
        }
        ASSERT_OK_AND_ASSIGN(int64_t first_snapshot_id,
                             Commit(first_progress, /*commit_identifier=*/1));
        ASSERT_OK(second_writer->RefreshCommittedSnapshot(first_snapshot_id));
        const std::vector<Row> rows_after_refresh =
            prepare_before_refresh ? std::vector<Row>{initial_rows[2], fourth_row}
                                   : std::vector<Row>{initial_rows[1], initial_rows[2], fourth_row};
        ASSERT_OK_AND_ASSIGN(std::vector<Row> refreshed_rows, ReadRows(realtime_context));
        ASSERT_EQ(rows_after_refresh, refreshed_rows);
        if (!prepare_before_refresh) {
            ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> second_delete,
                                 MakeBatch({initial_rows[1]}, /*partitioned=*/false, /*bucket=*/0,
                                           {RecordBatch::RowKind::DELETE}));
            ASSERT_OK(second_writer->Write(std::move(second_delete)));
            ASSERT_OK_AND_ASSIGN(second_progress,
                                 second_writer->PrepareCommitWithProgress(/*commit_identifier=*/2));
        }
        const std::vector<Row> expected_rows = {initial_rows[2], fourth_row};
        ASSERT_OK_AND_ASSIGN(std::vector<Row> prepared_rows, ReadRows(realtime_context));
        ASSERT_EQ(expected_rows, prepared_rows);
        ASSERT_OK(Commit(second_progress, /*commit_identifier=*/2));
        ASSERT_OK_AND_ASSIGN(std::vector<Row> disk_rows, ReadRows());
        ASSERT_EQ(expected_rows, disk_rows);

        // Another prepare must preserve deletions from the message prepared before refresh,
        // even though its commit has not yet been refreshed into this context.
        ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> third_delete,
                             MakeBatch({initial_rows[2]}, /*partitioned=*/false, /*bucket=*/0,
                                       {RecordBatch::RowKind::DELETE}));
        ASSERT_OK(second_writer->Write(std::move(third_delete)));
        ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> third_progress,
                             second_writer->PrepareCommitWithProgress(/*commit_identifier=*/3));
        ASSERT_OK_AND_ASSIGN(std::vector<Row> rows_before_commit, ReadRows(realtime_context));
        ASSERT_EQ(std::vector<Row>{fourth_row}, rows_before_commit);
        ASSERT_OK_AND_ASSIGN(int64_t final_snapshot_id,
                             Commit(third_progress, /*commit_identifier=*/3));
        ASSERT_OK_AND_ASSIGN(disk_rows, ReadRows());
        ASSERT_EQ(std::vector<Row>{fourth_row}, disk_rows);
        ASSERT_OK_AND_ASSIGN(std::vector<Row> rows_before_refresh, ReadRows(realtime_context));
        ASSERT_EQ(std::vector<Row>{fourth_row}, rows_before_refresh);
        ASSERT_OK(second_writer->RefreshCommittedSnapshot(final_snapshot_id));
        ASSERT_OK_AND_ASSIGN(std::vector<Row> final_rows, ReadRows(realtime_context));
        ASSERT_EQ(std::vector<Row>{fourth_row}, final_rows);
        ASSERT_OK(second_writer->Close());
    }
}

TEST_F(RealtimeWriteInteTest, TestRealtimeDeduplicateWithinOneBatch) {
    ConfigureRealtimeDeduplicateTable();
    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
                         RealtimeContext::Create());
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
                         CreateRealtimeWriter(realtime_context));

    ASSERT_OK_AND_ASSIGN(
        std::unique_ptr<RecordBatch> batch,
        MakeBatch({Row{1, "old-one", "p0"}, Row{2, "two", "p0"}, Row{1, "new-one", "p0"}},
                  /*partitioned=*/false));
    ASSERT_OK(writer->Write(std::move(batch)));
    const std::vector<Row> expected_rows = {{2, "two", "p0"}, {1, "new-one", "p0"}};
    ASSERT_OK_AND_ASSIGN(std::vector<Row> realtime_rows, ReadRows(realtime_context));
    ASSERT_EQ(expected_rows, realtime_rows);

    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> progress,
                         writer->PrepareCommitWithProgress(/*commit_identifier=*/0));
    ASSERT_EQ(1, progress.size());
    ASSERT_EQ(OffsetRange(0, 3), progress[0].offset_range);
    std::shared_ptr<CommitMessageImpl> message =
        std::dynamic_pointer_cast<CommitMessageImpl>(progress[0].commit_message);
    ASSERT_NE(nullptr, message);
    ASSERT_EQ(1, message->GetNewFilesIncrement().NewFiles().size());
    ASSERT_EQ(2, message->GetNewFilesIncrement().NewFiles()[0]->row_count);
    ASSERT_TRUE(message->GetCompactIncrement().NewIndexFiles().empty());

    ASSERT_OK_AND_ASSIGN(int64_t snapshot_id, Commit(progress, /*commit_identifier=*/0));
    ASSERT_OK(writer->RefreshCommittedSnapshot(snapshot_id));
    ASSERT_OK(writer->Close());
    writer.reset();
    realtime_context.reset();

    ASSERT_OK_AND_ASSIGN(std::vector<Row> disk_rows, ReadRows());
    ASSERT_EQ(expected_rows, disk_rows);
}

TEST_F(RealtimeWriteInteTest, TestRealtimeDeduplicateAcrossSealedSegments) {
    ConfigureRealtimeDeduplicateTable();
    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
                         RealtimeContext::Create());
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
                         CreateRealtimeWriter(realtime_context));

    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> first_batch,
                         MakeBatch({Row{1, "old-one", "p0"}, Row{2, "two", "p0"}},
                                   /*partitioned=*/false));
    ASSERT_OK(writer->Write(std::move(first_batch)));
    ASSERT_OK(writer->Seal());

    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> second_batch,
                         MakeBatch({Row{1, "new-one", "p0"}, Row{3, "three", "p0"}},
                                   /*partitioned=*/false));
    ASSERT_OK(writer->Write(std::move(second_batch)));
    const std::vector<Row> expected_rows = {
        {2, "two", "p0"}, {1, "new-one", "p0"}, {3, "three", "p0"}};
    ASSERT_OK_AND_ASSIGN(std::vector<Row> rows_before_prepare, ReadRows(realtime_context));
    ASSERT_EQ(expected_rows, rows_before_prepare);

    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> progress,
                         writer->PrepareCommitWithProgress(/*commit_identifier=*/0));
    ASSERT_EQ(1, progress.size());
    ASSERT_EQ(OffsetRange(0, 4), progress[0].offset_range);
    std::shared_ptr<CommitMessageImpl> message =
        std::dynamic_pointer_cast<CommitMessageImpl>(progress[0].commit_message);
    ASSERT_NE(nullptr, message);
    ASSERT_EQ(2, message->GetNewFilesIncrement().NewFiles().size());
    ASSERT_TRUE(message->GetCompactIncrement().NewIndexFiles().empty());
    ASSERT_NE(nullptr, progress[0].deleted_offsets);
    ASSERT_FALSE(progress[0].deleted_offsets->IsEmpty());

    ASSERT_OK_AND_ASSIGN(std::vector<Row> rows_after_prepare, ReadRows(realtime_context));
    ASSERT_EQ(expected_rows, rows_after_prepare);

    ASSERT_OK_AND_ASSIGN(int64_t snapshot_id, Commit(progress, /*commit_identifier=*/0));
    ASSERT_OK(writer->RefreshCommittedSnapshot(snapshot_id));
    ASSERT_OK_AND_ASSIGN(std::vector<Row> rows_after_refresh, ReadRows(realtime_context));
    ASSERT_EQ(expected_rows, rows_after_refresh);
    ASSERT_OK(writer->Close());
}

TEST_F(RealtimeWriteInteTest, TestRealtimeDeduplicateAcrossPreparedSegments) {
    ConfigureRealtimeDeduplicateTable();
    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
                         RealtimeContext::Create());
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
                         CreateRealtimeWriter(realtime_context));

    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> first_batch,
                         MakeBatch({Row{1, "old-one", "p0"}, Row{2, "two", "p0"}},
                                   /*partitioned=*/false));
    ASSERT_OK(writer->Write(std::move(first_batch)));
    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> progress,
                         writer->PrepareCommitWithProgress(/*commit_identifier=*/0));
    ASSERT_EQ(1, progress.size());
    ASSERT_EQ(OffsetRange(0, 2), progress[0].offset_range);
    ASSERT_NE(nullptr, progress[0].deleted_offsets);
    ASSERT_TRUE(progress[0].deleted_offsets->IsEmpty());

    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> second_batch,
                         MakeBatch({Row{1, "new-one", "p0"}, Row{3, "three", "p0"}},
                                   /*partitioned=*/false));
    ASSERT_OK(writer->Write(std::move(second_batch)));
    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> second_progress,
                         writer->PrepareCommitWithProgress(/*commit_identifier=*/1));
    ASSERT_EQ(1, second_progress.size());
    ASSERT_EQ(OffsetRange(2, 4), second_progress[0].offset_range);
    ASSERT_NE(nullptr, second_progress[0].deleted_offsets);
    ASSERT_EQ("{0}", second_progress[0].deleted_offsets->ToString());
    ASSERT_TRUE(progress[0].deleted_offsets->IsEmpty());
    std::shared_ptr<CommitMessageImpl> second_message =
        std::dynamic_pointer_cast<CommitMessageImpl>(second_progress[0].commit_message);
    ASSERT_NE(nullptr, second_message);
    ASSERT_EQ(1, second_message->GetNewFilesIncrement().NewFiles().size());
    ASSERT_TRUE(second_message->GetCompactIncrement().NewIndexFiles().empty());

    progress.push_back(std::move(second_progress[0]));
    ASSERT_OK_AND_ASSIGN(int64_t snapshot_id, Commit(progress, /*commit_identifier=*/1));
    const std::vector<Row> expected_rows = {
        {2, "two", "p0"}, {1, "new-one", "p0"}, {3, "three", "p0"}};
    ASSERT_OK_AND_ASSIGN(std::vector<Row> rows_before_refresh, ReadRows(realtime_context));
    ASSERT_EQ(expected_rows, rows_before_refresh);
    ASSERT_OK(writer->RefreshCommittedSnapshot(snapshot_id));
    ASSERT_OK_AND_ASSIGN(std::vector<Row> rows_after_refresh, ReadRows(realtime_context));
    ASSERT_EQ(expected_rows, rows_after_refresh);
    ASSERT_OK(writer->Close());
}

TEST_F(RealtimeWriteInteTest, TestRealtimeDeduplicateDeletesCommittedKey) {
    ConfigureRealtimeDeduplicateTable();
    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
                         RealtimeContext::Create());
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
                         CreateRealtimeWriter(realtime_context));

    ASSERT_OK_AND_ASSIGN(
        std::unique_ptr<RecordBatch> insert_batch,
        MakeBatch({Row{1, "one", "p0"}, Row{2, "two", "p0"}}, /*partitioned=*/false));
    ASSERT_OK(writer->Write(std::move(insert_batch)));
    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> insert_progress,
                         writer->PrepareCommitWithProgress(/*commit_identifier=*/0));
    ASSERT_OK_AND_ASSIGN(int64_t insert_snapshot_id,
                         Commit(insert_progress, /*commit_identifier=*/0));
    ASSERT_OK(writer->RefreshCommittedSnapshot(insert_snapshot_id));

    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> delete_batch,
                         MakeBatch({Row{1, "ignored", "p0"}}, /*partitioned=*/false, /*bucket=*/0,
                                   {RecordBatch::RowKind::DELETE}));
    ASSERT_OK(writer->Write(std::move(delete_batch)));
    const std::vector<Row> expected_rows = {{2, "two", "p0"}};
    ASSERT_OK_AND_ASSIGN(std::vector<Row> realtime_rows, ReadRows(realtime_context));
    ASSERT_EQ(expected_rows, realtime_rows);

    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> delete_progress,
                         writer->PrepareCommitWithProgress(/*commit_identifier=*/1));
    ASSERT_EQ(1, delete_progress.size());
    ASSERT_EQ(OffsetRange(2, 3), delete_progress[0].offset_range);
    std::shared_ptr<CommitMessageImpl> delete_message =
        std::dynamic_pointer_cast<CommitMessageImpl>(delete_progress[0].commit_message);
    ASSERT_NE(nullptr, delete_message);
    ASSERT_TRUE(delete_message->GetNewFilesIncrement().NewFiles().empty());
    ASSERT_TRUE(delete_message->GetCompactIncrement().NewIndexFiles().empty());
    ASSERT_NE(nullptr, delete_progress[0].deleted_offsets);
    ASSERT_FALSE(delete_progress[0].deleted_offsets->IsEmpty());

    ASSERT_OK_AND_ASSIGN(int64_t delete_snapshot_id,
                         Commit(delete_progress, /*commit_identifier=*/1));
    ASSERT_OK(writer->RefreshCommittedSnapshot(delete_snapshot_id));
    ASSERT_OK(writer->Close());
    writer.reset();
    realtime_context.reset();

    ASSERT_OK_AND_ASSIGN(std::vector<Row> disk_rows, ReadRows());
    ASSERT_EQ(expected_rows, disk_rows);
}

TEST_F(RealtimeWriteInteTest, TestRealtimeDeduplicateInsertThenDeleteInOneSegment) {
    ConfigureRealtimeDeduplicateTable();
    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
                         RealtimeContext::Create());
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
                         CreateRealtimeWriter(realtime_context));

    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> batch,
                         MakeBatch({Row{1, "one", "p0"}, Row{1, "ignored", "p0"}},
                                   /*partitioned=*/false, /*bucket=*/0,
                                   {RecordBatch::RowKind::INSERT, RecordBatch::RowKind::DELETE}));
    ASSERT_OK(writer->Write(std::move(batch)));
    ASSERT_OK_AND_ASSIGN(std::vector<Row> realtime_rows, ReadRows(realtime_context));
    ASSERT_TRUE(realtime_rows.empty());

    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> progress,
                         writer->PrepareCommitWithProgress(/*commit_identifier=*/0));
    ASSERT_EQ(1, progress.size());
    ASSERT_EQ(OffsetRange(0, 2), progress[0].offset_range);
    std::shared_ptr<CommitMessageImpl> message =
        std::dynamic_pointer_cast<CommitMessageImpl>(progress[0].commit_message);
    ASSERT_NE(nullptr, message);
    ASSERT_TRUE(message->IsEmpty());

    ASSERT_OK_AND_ASSIGN(int64_t snapshot_id, Commit(progress, /*commit_identifier=*/0));
    ASSERT_OK(writer->RefreshCommittedSnapshot(snapshot_id));
    ASSERT_OK_AND_ASSIGN(RealtimeOffsetMap committed_offsets, ReadCommittedOffsets());
    ASSERT_EQ(2, committed_offsets.at(RealtimePartitionBucket(/*partition=*/{}, /*bucket=*/0)));
    ASSERT_OK_AND_ASSIGN(std::vector<Row> rows_after_refresh, ReadRows(realtime_context));
    ASSERT_TRUE(rows_after_refresh.empty());
    ASSERT_OK(writer->Close());
}

TEST_F(RealtimeWriteInteTest, TestRealtimeDeduplicateUpdateBeforeAndAfter) {
    ConfigureRealtimeDeduplicateTable();
    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
                         RealtimeContext::Create());
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
                         CreateRealtimeWriter(realtime_context));

    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> insert_batch,
                         MakeBatch({Row{1, "old-one", "p0"}, Row{2, "two", "p0"}},
                                   /*partitioned=*/false));
    ASSERT_OK(writer->Write(std::move(insert_batch)));
    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> insert_progress,
                         writer->PrepareCommitWithProgress(/*commit_identifier=*/0));
    ASSERT_OK_AND_ASSIGN(int64_t insert_snapshot_id,
                         Commit(insert_progress, /*commit_identifier=*/0));
    ASSERT_OK(writer->RefreshCommittedSnapshot(insert_snapshot_id));

    ASSERT_OK_AND_ASSIGN(
        std::unique_ptr<RecordBatch> update_batch,
        MakeBatch({Row{1, "ignored-before", "p0"}, Row{1, "new-one", "p0"}},
                  /*partitioned=*/false, /*bucket=*/0,
                  {RecordBatch::RowKind::UPDATE_BEFORE, RecordBatch::RowKind::UPDATE_AFTER}));
    ASSERT_OK(writer->Write(std::move(update_batch)));
    const std::vector<Row> expected_rows = {{2, "two", "p0"}, {1, "new-one", "p0"}};
    ASSERT_OK_AND_ASSIGN(std::vector<Row> realtime_rows, ReadRows(realtime_context));
    ASSERT_EQ(expected_rows, realtime_rows);

    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> update_progress,
                         writer->PrepareCommitWithProgress(/*commit_identifier=*/1));
    ASSERT_EQ(1, update_progress.size());
    ASSERT_EQ(OffsetRange(2, 4), update_progress[0].offset_range);
    std::shared_ptr<CommitMessageImpl> update_message =
        std::dynamic_pointer_cast<CommitMessageImpl>(update_progress[0].commit_message);
    ASSERT_NE(nullptr, update_message);
    ASSERT_EQ(1, update_message->GetNewFilesIncrement().NewFiles().size());
    ASSERT_TRUE(update_message->GetCompactIncrement().NewIndexFiles().empty());
    ASSERT_NE(nullptr, update_progress[0].deleted_offsets);
    ASSERT_FALSE(update_progress[0].deleted_offsets->IsEmpty());

    ASSERT_OK_AND_ASSIGN(int64_t update_snapshot_id,
                         Commit(update_progress, /*commit_identifier=*/1));
    ASSERT_OK(writer->RefreshCommittedSnapshot(update_snapshot_id));
    ASSERT_OK(writer->Close());
    writer.reset();
    realtime_context.reset();

    ASSERT_OK_AND_ASSIGN(std::vector<Row> disk_rows, ReadRows());
    ASSERT_EQ(expected_rows, disk_rows);
}

class RealtimeDeduplicateExactIndexInteTest : public RealtimeWriteInteTest,
                                              public ::testing::WithParamInterface<std::string> {};

TEST_P(RealtimeDeduplicateExactIndexInteTest,
       TestRealtimeDeduplicateAcrossMultipleFileIndexStorageForms) {
    options_[Options::FILE_INDEX_IN_MANIFEST_THRESHOLD] = "1MB";
    ConfigureRealtimeDeduplicateTable({}, GetParam());
    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
                         RealtimeContext::Create());

    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> embedded_writer,
                         CreateRealtimeWriter(realtime_context));
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> embedded_batch,
                         MakeBatch({Row{1, "old-one", "p0"}, Row{2, "two", "p0"}},
                                   /*partitioned=*/false));
    ASSERT_OK(embedded_writer->Write(std::move(embedded_batch)));
    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> embedded_progress,
                         embedded_writer->PrepareCommitWithProgress(/*commit_identifier=*/0));
    std::vector<std::shared_ptr<DataFileMeta>> embedded_files = NewFiles(embedded_progress);
    ASSERT_EQ(1, embedded_files.size());
    ASSERT_NE(nullptr, embedded_files[0]->embedded_index);
    // The only extra file is the deduplicate key-offset sidecar.
    ASSERT_EQ(1, embedded_files[0]->extra_files.size());
    ASSERT_OK_AND_ASSIGN(int64_t embedded_snapshot_id,
                         Commit(embedded_progress, /*commit_identifier=*/0));
    ASSERT_OK(embedded_writer->RefreshCommittedSnapshot(embedded_snapshot_id));
    ASSERT_OK(embedded_writer->Close());

    options_[Options::FILE_INDEX_IN_MANIFEST_THRESHOLD] = "1B";
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> external_writer,
                         CreateRealtimeWriter(realtime_context));
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> external_batch,
                         MakeBatch({Row{3, "old-three", "p0"}, Row{4, "four", "p0"}},
                                   /*partitioned=*/false));
    ASSERT_OK(external_writer->Write(std::move(external_batch)));
    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> external_progress,
                         external_writer->PrepareCommitWithProgress(/*commit_identifier=*/1));
    std::vector<std::shared_ptr<DataFileMeta>> external_files = NewFiles(external_progress);
    ASSERT_EQ(1, external_files.size());
    ASSERT_EQ(nullptr, external_files[0]->embedded_index);
    // The external file index precedes the deduplicate key-offset sidecar.
    ASSERT_EQ(2, external_files[0]->extra_files.size());
    ASSERT_OK_AND_ASSIGN(int64_t external_snapshot_id,
                         Commit(external_progress, /*commit_identifier=*/1));
    ASSERT_OK(external_writer->RefreshCommittedSnapshot(external_snapshot_id));

    ASSERT_OK_AND_ASSIGN(
        std::unique_ptr<RecordBatch> update_batch,
        MakeBatch({Row{1, "new-one", "p0"}, Row{3, "new-three", "p0"}, Row{5, "five", "p0"}},
                  /*partitioned=*/false));
    ASSERT_OK(external_writer->Write(std::move(update_batch)));
    const std::vector<Row> expected_rows = {{2, "two", "p0"},
                                            {4, "four", "p0"},
                                            {1, "new-one", "p0"},
                                            {3, "new-three", "p0"},
                                            {5, "five", "p0"}};
    ASSERT_OK_AND_ASSIGN(std::vector<Row> rows_before_prepare, ReadRows(realtime_context));
    ASSERT_EQ(expected_rows, rows_before_prepare);

    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> update_progress,
                         external_writer->PrepareCommitWithProgress(/*commit_identifier=*/2));
    ASSERT_EQ(1, update_progress.size());
    std::shared_ptr<CommitMessageImpl> update_message =
        std::dynamic_pointer_cast<CommitMessageImpl>(update_progress[0].commit_message);
    ASSERT_NE(nullptr, update_message);
    ASSERT_EQ(1, update_message->GetNewFilesIncrement().NewFiles().size());
    ASSERT_TRUE(update_message->GetCompactIncrement().NewIndexFiles().empty());
    ASSERT_NE(nullptr, update_progress[0].deleted_offsets);
    ASSERT_FALSE(update_progress[0].deleted_offsets->IsEmpty());
    ASSERT_OK_AND_ASSIGN(std::vector<Row> rows_after_prepare, ReadRows(realtime_context));
    ASSERT_EQ(expected_rows, rows_after_prepare);

    ASSERT_OK_AND_ASSIGN(int64_t update_snapshot_id,
                         Commit(update_progress, /*commit_identifier=*/2));
    ASSERT_OK(external_writer->RefreshCommittedSnapshot(update_snapshot_id));
    ASSERT_OK(external_writer->Close());
    external_writer.reset();
    realtime_context.reset();

    ASSERT_OK_AND_ASSIGN(std::vector<Row> disk_rows, ReadRows());
    ASSERT_EQ(expected_rows, disk_rows);
}

INSTANTIATE_TEST_SUITE_P(ExactIndex, RealtimeDeduplicateExactIndexInteTest,
                         ::testing::Values("bitmap", "bsi", "range-bitmap"));

TEST_F(RealtimeWriteInteTest, TestRealtimeDeduplicateReadViewIsStableAcrossWriteAndRefresh) {
    ConfigureRealtimeDeduplicateTable();
    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
                         RealtimeContext::Create());
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
                         CreateRealtimeWriter(realtime_context));

    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> base_batch,
                         MakeBatch({Row{1, "old-one", "p0"}, Row{2, "old-two", "p0"}},
                                   /*partitioned=*/false));
    ASSERT_OK(writer->Write(std::move(base_batch)));
    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> base_progress,
                         writer->PrepareCommitWithProgress(/*commit_identifier=*/0));
    ASSERT_OK_AND_ASSIGN(int64_t base_snapshot_id, Commit(base_progress, /*commit_identifier=*/0));
    ASSERT_OK(writer->RefreshCommittedSnapshot(base_snapshot_id));

    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> first_update,
                         MakeBatch({Row{1, "new-one", "p0"}, Row{3, "three", "p0"}},
                                   /*partitioned=*/false));
    ASSERT_OK(writer->Write(std::move(first_update)));
    ASSERT_OK_AND_ASSIGN(std::shared_ptr<Plan> old_plan,
                         CreatePlan(realtime_context, /*predicate=*/nullptr));
    ASSERT_OK_AND_ASSIGN(std::shared_ptr<Plan> old_plan_across_refresh,
                         CreatePlan(realtime_context, /*predicate=*/nullptr));

    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> second_update,
                         MakeBatch({Row{2, "new-two", "p0"}, Row{4, "four", "p0"}},
                                   /*partitioned=*/false));
    ASSERT_OK(writer->Write(std::move(second_update)));

    const std::vector<Row> old_expected_rows = {
        {2, "old-two", "p0"}, {1, "new-one", "p0"}, {3, "three", "p0"}};
    ASSERT_OK_AND_ASSIGN(std::vector<Row> old_plan_rows, ReadRows(old_plan, realtime_context));
    ASSERT_EQ(old_expected_rows, old_plan_rows);

    const std::vector<Row> latest_expected_rows = {
        {1, "new-one", "p0"}, {3, "three", "p0"}, {2, "new-two", "p0"}, {4, "four", "p0"}};
    ASSERT_OK_AND_ASSIGN(std::vector<Row> latest_rows_before_prepare, ReadRows(realtime_context));
    ASSERT_EQ(latest_expected_rows, latest_rows_before_prepare);

    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> progress,
                         writer->PrepareCommitWithProgress(/*commit_identifier=*/1));
    ASSERT_OK_AND_ASSIGN(int64_t snapshot_id, Commit(progress, /*commit_identifier=*/1));
    ASSERT_OK(writer->RefreshCommittedSnapshot(snapshot_id));

    ASSERT_OK_AND_ASSIGN(std::vector<Row> pinned_rows_after_refresh,
                         ReadRows(old_plan_across_refresh, realtime_context));
    ASSERT_EQ(old_expected_rows, pinned_rows_after_refresh);
    ASSERT_OK_AND_ASSIGN(std::vector<Row> latest_rows_after_refresh, ReadRows(realtime_context));
    ASSERT_EQ(latest_expected_rows, latest_rows_after_refresh);
    ASSERT_OK(writer->Close());
}

TEST_F(RealtimeWriteInteTest, TestRealtimeDeduplicateIsolatesPartitionsAndBuckets) {
    options_[Options::BUCKET] = "2";
    ConfigureRealtimeDeduplicateTable(/*partition_keys=*/{"pt"});
    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
                         RealtimeContext::Create());
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
                         CreateRealtimeWriter(realtime_context));

    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> p0b0_batch,
                         MakeBatch({Row{1, "p0-b0-old", "p0"}, Row{2, "p0-b0-keep", "p0"}},
                                   /*partitioned=*/true, /*bucket=*/0));
    ASSERT_OK(writer->Write(std::move(p0b0_batch)));
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> p0b1_batch,
                         MakeBatch({Row{1, "p0-b1-old", "p0"}, Row{3, "p0-b1-keep", "p0"}},
                                   /*partitioned=*/true, /*bucket=*/1));
    ASSERT_OK(writer->Write(std::move(p0b1_batch)));
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> p1b0_batch,
                         MakeBatch({Row{1, "p1-b0-old", "p1"}, Row{4, "p1-b0-keep", "p1"}},
                                   /*partitioned=*/true, /*bucket=*/0));
    ASSERT_OK(writer->Write(std::move(p1b0_batch)));
    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> base_progress,
                         writer->PrepareCommitWithProgress(/*commit_identifier=*/0));
    ASSERT_EQ(3, base_progress.size());
    ASSERT_OK_AND_ASSIGN(int64_t base_snapshot_id, Commit(base_progress, /*commit_identifier=*/0));
    ASSERT_OK(writer->RefreshCommittedSnapshot(base_snapshot_id));

    ASSERT_OK_AND_ASSIGN(
        std::unique_ptr<RecordBatch> update_batch,
        MakeBatch({Row{1, "p0-b0-new", "p0"}}, /*partitioned=*/true, /*bucket=*/0));
    ASSERT_OK(writer->Write(std::move(update_batch)));
    const std::vector<Row> expected_rows = {{2, "p0-b0-keep", "p0"}, {1, "p0-b0-new", "p0"},
                                            {1, "p0-b1-old", "p0"},  {3, "p0-b1-keep", "p0"},
                                            {1, "p1-b0-old", "p1"},  {4, "p1-b0-keep", "p1"}};
    ASSERT_OK_AND_ASSIGN(std::vector<Row> rows_before_prepare, ReadRows(realtime_context));
    ASSERT_EQ(std::multiset<Row>(expected_rows.begin(), expected_rows.end()),
              std::multiset<Row>(rows_before_prepare.begin(), rows_before_prepare.end()));

    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> update_progress,
                         writer->PrepareCommitWithProgress(/*commit_identifier=*/1));
    ASSERT_EQ(1, update_progress.size());
    ASSERT_EQ(RealtimePartitionBucket({{"pt", "p0"}}, /*bucket=*/0),
              update_progress[0].partition_bucket);
    ASSERT_EQ(OffsetRange(2, 3), update_progress[0].offset_range);
    std::shared_ptr<CommitMessageImpl> update_message =
        std::dynamic_pointer_cast<CommitMessageImpl>(update_progress[0].commit_message);
    ASSERT_NE(nullptr, update_message);
    ASSERT_TRUE(update_message->GetCompactIncrement().NewIndexFiles().empty());
    ASSERT_NE(nullptr, update_progress[0].deleted_offsets);
    ASSERT_FALSE(update_progress[0].deleted_offsets->IsEmpty());

    ASSERT_OK_AND_ASSIGN(std::vector<Row> rows_after_prepare, ReadRows(realtime_context));
    ASSERT_EQ(std::multiset<Row>(expected_rows.begin(), expected_rows.end()),
              std::multiset<Row>(rows_after_prepare.begin(), rows_after_prepare.end()));

    ASSERT_OK_AND_ASSIGN(int64_t update_snapshot_id,
                         Commit(update_progress, /*commit_identifier=*/1));
    ASSERT_OK_AND_ASSIGN(std::vector<Row> rows_before_refresh, ReadRows(realtime_context));
    ASSERT_EQ(std::multiset<Row>(expected_rows.begin(), expected_rows.end()),
              std::multiset<Row>(rows_before_refresh.begin(), rows_before_refresh.end()));
    ASSERT_OK(writer->RefreshCommittedSnapshot(update_snapshot_id));
    ASSERT_OK_AND_ASSIGN(std::vector<Row> rows_after_refresh, ReadRows(realtime_context));
    ASSERT_EQ(std::multiset<Row>(expected_rows.begin(), expected_rows.end()),
              std::multiset<Row>(rows_after_refresh.begin(), rows_after_refresh.end()));

    ASSERT_OK_AND_ASSIGN(RealtimeOffsetMap committed_offsets, ReadCommittedOffsets());
    ASSERT_EQ(3, committed_offsets.size());
    ASSERT_EQ(3, committed_offsets.at(RealtimePartitionBucket({{"pt", "p0"}}, /*bucket=*/0)));
    ASSERT_EQ(2, committed_offsets.at(RealtimePartitionBucket({{"pt", "p0"}}, /*bucket=*/1)));
    ASSERT_EQ(2, committed_offsets.at(RealtimePartitionBucket({{"pt", "p1"}}, /*bucket=*/0)));
    ASSERT_OK(writer->Close());
}

TEST_F(RealtimeWriteInteTest,
       TestRealtimeDeduplicatePredicatePushdownAcrossDataFileSealedAndBuilding) {
    options_[Options::WRITE_BATCH_SIZE] = "1";
    options_[Options::REALTIME_STORE_STATS_MODE] = "full";
    options_["parquet.page.size"] = "1";
    options_["parquet.enable.dictionary"] = "false";
    options_["parquet.write.enable-page-index"] = "true";
    options_["parquet.write.max-row-group-length"] = "1";
    options_["parquet.read.enable-page-index-filter"] = "true";
    ConfigureRealtimeDeduplicateTable();
    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
                         RealtimeContext::Create());
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
                         CreateRealtimeWriter(realtime_context));

    auto write_one = [&](const Row& row) -> Status {
        PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<RecordBatch> batch,
                               MakeBatch({row}, /*partitioned=*/false));
        return writer->Write(std::move(batch));
    };

    ASSERT_OK(write_one({1, "match", "p0"}));
    ASSERT_OK(write_one({2, "data-file-other", "p0"}));
    ASSERT_OK(write_one({7, "match", "p0"}));
    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> data_file_progress,
                         writer->PrepareCommitWithProgress(/*commit_identifier=*/0));
    ASSERT_OK_AND_ASSIGN(int64_t data_file_snapshot_id,
                         Commit(data_file_progress, /*commit_identifier=*/0));
    ASSERT_OK(writer->RefreshCommittedSnapshot(data_file_snapshot_id));

    ASSERT_OK(write_one({3, "match", "p0"}));
    ASSERT_OK(write_one({4, "sealed-other", "p0"}));
    ASSERT_OK(write_one({7, "replacement-other", "p0"}));
    ASSERT_OK(writer->Seal());

    ASSERT_OK(write_one({5, "match", "p0"}));
    ASSERT_OK(write_one({6, "building-other", "p0"}));

    const std::string matching_payload = "match";
    std::shared_ptr<Predicate> scan_predicate = PredicateBuilder::Equal(
        /*field_index=*/2, /*field_name=*/"payload", FieldType::STRING,
        Literal(FieldType::STRING, matching_payload.data(), matching_payload.size()));
    std::shared_ptr<Predicate> read_predicate = PredicateBuilder::Equal(
        /*field_index=*/1, /*field_name=*/"payload", FieldType::STRING,
        Literal(FieldType::STRING, matching_payload.data(), matching_payload.size()));
    ASSERT_OK_AND_ASSIGN(std::shared_ptr<Plan> plan, CreatePlan(realtime_context, scan_predicate));

    // Disable residual filtering so the result proves that the predicate reached the data file
    // reader and both the sealed and building in-memory segments.
    ASSERT_OK_AND_ASSIGN(std::vector<Row> rows, ReadRows(plan, realtime_context, read_predicate,
                                                         /*enable_predicate_filter=*/false));
    ASSERT_EQ((std::vector<Row>{{1, "match", "p0"}, {3, "match", "p0"}, {5, "match", "p0"}}), rows);
    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> final_progress,
                         writer->PrepareCommitWithProgress(/*commit_identifier=*/1));
    ASSERT_EQ(1, final_progress.size());
    std::shared_ptr<CommitMessageImpl> final_message =
        std::dynamic_pointer_cast<CommitMessageImpl>(final_progress[0].commit_message);
    ASSERT_NE(nullptr, final_message);
    ASSERT_TRUE(final_message->GetCompactIncrement().NewIndexFiles().empty());
    ASSERT_NE(nullptr, final_progress[0].deleted_offsets);
    ASSERT_FALSE(final_progress[0].deleted_offsets->IsEmpty());
    ASSERT_OK(writer->Close());
}

TEST_F(RealtimeWriteInteTest, TestRealtimeReadBatchSizeSplitsLargeAppendBatch) {
    CreateTable(/*partition_keys=*/{});
    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
                         RealtimeContext::Create());
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
                         CreateRealtimeWriter(realtime_context));

    constexpr int64_t kRowCount = 1000;
    const std::vector<Row> expected_rows = MakeRows(/*first_id=*/0, kRowCount, /*partition=*/"p0");
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> batch,
                         MakeBatch(expected_rows, /*partitioned=*/false));
    ASSERT_OK(writer->Write(std::move(batch)));

    for (int32_t read_batch_size : {1, 17, 128, 2048}) {
        options_[Options::READ_BATCH_SIZE] = std::to_string(read_batch_size);
        ASSERT_OK_AND_ASSIGN(std::vector<Row> actual_rows, ReadRows(realtime_context));
        ASSERT_EQ(expected_rows, actual_rows);

        ASSERT_OK_AND_ASSIGN(std::shared_ptr<Plan> plan,
                             CreatePlan(realtime_context, /*predicate=*/nullptr));
        ASSERT_OK_AND_ASSIGN(
            std::shared_ptr<arrow::ChunkedArray> result,
            ReadPlan(plan, realtime_context, {"id", "payload", "pt"}, /*predicate=*/nullptr,
                     /*enable_predicate_filter=*/false));
        ASSERT_NE(nullptr, result);
        ASSERT_EQ(kRowCount, result->length());
        ASSERT_EQ((kRowCount + read_batch_size - 1) / read_batch_size, result->num_chunks());
        for (const std::shared_ptr<arrow::Array>& chunk : result->chunks()) {
            ASSERT_GT(chunk->length(), 0);
            ASSERT_LE(chunk->length(), read_batch_size);
        }
    }
    FinalizeCommitAndCheck(writer.get(), /*realtime_commits=*/{}, /*prepare_identifier=*/0,
                           expected_rows);
}

TEST_F(RealtimeWriteInteTest, TestRealtimeMetricsTrackBuildingSealedAndCommittedData) {
    CreateTable(/*partition_keys=*/{});
    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
                         RealtimeContext::Create());
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
                         CreateRealtimeWriter(realtime_context));

    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> first_batch,
                         MakeBatch(MakeRows(/*first_id=*/0, /*count=*/3, /*partition=*/"p0"),
                                   /*partitioned=*/false));
    ASSERT_OK(writer->Write(std::move(first_batch)));
    ASSERT_OK_AND_ASSIGN(RealtimeMetricValues building_metrics,
                         ReadRealtimeMetrics(writer->GetMetrics()));
    ASSERT_GT(building_metrics.building_memory, 0);
    ASSERT_EQ(0, building_metrics.sealed_memory);
    ASSERT_EQ(building_metrics.building_memory, building_metrics.total_memory);
    ASSERT_EQ(3, building_metrics.building_rows);
    ASSERT_EQ(0, building_metrics.sealed_rows);
    ASSERT_EQ(3, building_metrics.total_rows);

    ASSERT_OK(writer->Seal());
    ASSERT_OK_AND_ASSIGN(RealtimeMetricValues sealed_metrics,
                         ReadRealtimeMetrics(realtime_context->GetMetrics()));
    ASSERT_EQ(0, sealed_metrics.building_memory);
    ASSERT_EQ(building_metrics.total_memory, sealed_metrics.sealed_memory);
    ASSERT_EQ(sealed_metrics.sealed_memory, sealed_metrics.total_memory);
    ASSERT_EQ(0, sealed_metrics.building_rows);
    ASSERT_EQ(3, sealed_metrics.sealed_rows);
    ASSERT_EQ(3, sealed_metrics.total_rows);

    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> second_batch,
                         MakeBatch(MakeRows(/*first_id=*/3, /*count=*/2, /*partition=*/"p0"),
                                   /*partitioned=*/false));
    ASSERT_OK(writer->Write(std::move(second_batch)));
    ASSERT_OK_AND_ASSIGN(RealtimeMetricValues mixed_metrics,
                         ReadRealtimeMetrics(realtime_context->GetMetrics()));
    ASSERT_GT(mixed_metrics.building_memory, 0);
    ASSERT_EQ(sealed_metrics.sealed_memory, mixed_metrics.sealed_memory);
    ASSERT_EQ(mixed_metrics.building_memory + mixed_metrics.sealed_memory,
              mixed_metrics.total_memory);
    ASSERT_EQ(2, mixed_metrics.building_rows);
    ASSERT_EQ(3, mixed_metrics.sealed_rows);
    ASSERT_EQ(5, mixed_metrics.total_rows);

    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> commits,
                         writer->PrepareCommitWithProgress(/*commit_identifier=*/0));
    ASSERT_EQ(1, commits.size());
    ASSERT_OK_AND_ASSIGN(RealtimeMetricValues prepared_metrics,
                         ReadRealtimeMetrics(writer->GetMetrics()));
    ASSERT_EQ(0, prepared_metrics.building_memory);
    ASSERT_EQ(mixed_metrics.total_memory, prepared_metrics.sealed_memory);
    ASSERT_EQ(prepared_metrics.sealed_memory, prepared_metrics.total_memory);
    ASSERT_EQ(0, prepared_metrics.building_rows);
    ASSERT_EQ(5, prepared_metrics.sealed_rows);
    ASSERT_EQ(5, prepared_metrics.total_rows);

    ASSERT_OK_AND_ASSIGN(int64_t snapshot_id, Commit(commits, /*commit_identifier=*/0));
    ASSERT_OK(writer->RefreshCommittedSnapshot(snapshot_id));
    ASSERT_OK_AND_ASSIGN(RealtimeMetricValues reclaimed_metrics,
                         ReadRealtimeMetrics(realtime_context->GetMetrics()));
    ASSERT_EQ(0, reclaimed_metrics.building_memory);
    ASSERT_EQ(0, reclaimed_metrics.sealed_memory);
    ASSERT_EQ(0, reclaimed_metrics.total_memory);
    ASSERT_EQ(0, reclaimed_metrics.building_rows);
    ASSERT_EQ(0, reclaimed_metrics.sealed_rows);
    ASSERT_EQ(0, reclaimed_metrics.total_rows);
    ASSERT_OK(writer->Close());
}

TEST_F(RealtimeWriteInteTest, TestRealtimeCommitRetriesSnapshotConflict) {
    options_[Options::BUCKET] = "2";
    options_[Options::COMMIT_MAX_RETRIES] = "1";
    options_[Options::COMMIT_MIN_RETRY_WAIT] = "1ms";
    options_[Options::COMMIT_MAX_RETRY_WAIT] = "1ms";
    CreateTable(/*partition_keys=*/{});

    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> bucket0_writer, CreateRealtimeWriter());
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> bucket0_batch,
                         MakeBatch(MakeRows(/*first_id=*/0, /*count=*/2, /*partition=*/"p0"),
                                   /*partitioned=*/false, /*bucket=*/0));
    ASSERT_OK(bucket0_writer->Write(std::move(bucket0_batch)));
    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> bucket0_commits,
                         bucket0_writer->PrepareCommitWithProgress(/*commit_identifier=*/0));
    ASSERT_EQ(1, bucket0_commits.size());

    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> bucket1_writer, CreateRealtimeWriter());
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> bucket1_batch,
                         MakeBatch(MakeRows(/*first_id=*/10, /*count=*/3, /*partition=*/"p0"),
                                   /*partitioned=*/false, /*bucket=*/1));
    ASSERT_OK(bucket1_writer->Write(std::move(bucket1_batch)));
    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> bucket1_commits,
                         bucket1_writer->PrepareCommitWithProgress(/*commit_identifier=*/0));
    ASSERT_EQ(1, bucket1_commits.size());

    const std::string first_snapshot_path = PathUtil::JoinPath(table_path_, "snapshot/snapshot-1");
    auto conflict_file_system = std::make_shared<SnapshotConflictFileSystem>(first_snapshot_path);
    CommitContextBuilder bucket0_builder(table_path_, "bucket0_realtime_commit_user");
    ASSERT_OK_AND_ASSIGN(
        std::unique_ptr<CommitContext> bucket0_context,
        bucket0_builder.SetOptions(options_).WithFileSystem(conflict_file_system).Finish());
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreCommit> bucket0_commit,
                         FileStoreCommit::Create(std::move(bucket0_context)));
    CommitContextBuilder bucket1_builder(table_path_, "bucket1_realtime_commit_user");
    ASSERT_OK_AND_ASSIGN(
        std::unique_ptr<CommitContext> bucket1_context,
        bucket1_builder.SetOptions(options_).WithFileSystem(conflict_file_system).Finish());
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreCommit> bucket1_commit,
                         FileStoreCommit::Create(std::move(bucket1_context)));

    std::optional<Result<int64_t>> bucket0_result;
    std::optional<Result<int64_t>> bucket1_result;
    std::thread bucket0_thread([&]() {
        bucket0_result.emplace(bucket0_commit->CommitWithProgress(
            bucket0_commits, /*commit_identifier=*/0, /*watermark=*/std::nullopt));
    });
    std::thread bucket1_thread([&]() {
        bucket1_result.emplace(bucket1_commit->CommitWithProgress(
            bucket1_commits, /*commit_identifier=*/0, /*watermark=*/std::nullopt));
    });
    bucket0_thread.join();
    bucket1_thread.join();

    ASSERT_TRUE(bucket0_result.has_value());
    ASSERT_TRUE(bucket1_result.has_value());
    ASSERT_OK_AND_ASSIGN(int64_t bucket0_snapshot_id, std::move(bucket0_result.value()));
    ASSERT_OK_AND_ASSIGN(int64_t bucket1_snapshot_id, std::move(bucket1_result.value()));
    std::vector<int64_t> snapshot_ids = {bucket0_snapshot_id, bucket1_snapshot_id};
    std::sort(snapshot_ids.begin(), snapshot_ids.end());
    ASSERT_EQ((std::vector<int64_t>{1, 2}), snapshot_ids);
    ASSERT_EQ(2, conflict_file_system->ConflictSnapshotAttempts());

    ASSERT_OK_AND_ASSIGN(uint64_t bucket0_attempts, bucket0_commit->GetCommitMetrics()->GetCounter(
                                                        CommitMetrics::LAST_COMMIT_ATTEMPTS));
    ASSERT_OK_AND_ASSIGN(uint64_t bucket1_attempts, bucket1_commit->GetCommitMetrics()->GetCounter(
                                                        CommitMetrics::LAST_COMMIT_ATTEMPTS));
    std::vector<uint64_t> attempts = {bucket0_attempts, bucket1_attempts};
    std::sort(attempts.begin(), attempts.end());
    ASSERT_EQ((std::vector<uint64_t>{1, 2}), attempts);

    ASSERT_OK_AND_ASSIGN(RealtimeOffsetMap committed_offsets, ReadCommittedOffsets());
    ASSERT_EQ(2, committed_offsets.size());
    ASSERT_EQ(2, committed_offsets.at(RealtimePartitionBucket(/*partition=*/{}, /*bucket=*/0)));
    ASSERT_EQ(3, committed_offsets.at(RealtimePartitionBucket(/*partition=*/{}, /*bucket=*/1)));
    ASSERT_OK_AND_ASSIGN(std::vector<Row> rows, ReadRows());
    std::sort(rows.begin(), rows.end());
    std::vector<Row> expected_rows = MakeRows(/*first_id=*/0, /*count=*/2, /*partition=*/"p0");
    std::vector<Row> bucket1_rows = MakeRows(/*first_id=*/10, /*count=*/3, /*partition=*/"p0");
    expected_rows.insert(expected_rows.end(), bucket1_rows.begin(), bucket1_rows.end());
    std::sort(expected_rows.begin(), expected_rows.end());
    ASSERT_EQ(expected_rows, rows);
    ASSERT_OK(bucket0_writer->Close());
    ASSERT_OK(bucket1_writer->Close());
}

TEST_F(RealtimeWriteInteTest, TestRealtimeCommitRetriesConflictWithCompaction) {
    options_[Options::COMMIT_MAX_RETRIES] = "1";
    options_[Options::COMMIT_MIN_RETRY_WAIT] = "1ms";
    options_[Options::COMMIT_MAX_RETRY_WAIT] = "1ms";
    CreatePkTable();
    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
                         RealtimeContext::Create());
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
                         CreateRealtimeWriter(realtime_context));

    for (int64_t id = 0; id < 2; ++id) {
        ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> batch,
                             MakeBatch(MakeRows(id, /*count=*/1, /*partition=*/"p0"),
                                       /*partitioned=*/false));
        ASSERT_OK(writer->Write(std::move(batch)));
        ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> progress,
                             writer->PrepareCommitWithProgress(/*commit_identifier=*/id));
        ASSERT_OK_AND_ASSIGN(int64_t snapshot_id, Commit(progress, /*commit_identifier=*/id));
        ASSERT_OK(writer->RefreshCommittedSnapshot(snapshot_id));
    }

    WriteContextBuilder compact_write_builder(table_path_, "compaction_write_user");
    compact_write_builder.SetOptions(options_).WithStreamingMode(true).WithTempDirectory(
        PathUtil::JoinPath(dir_->Str(), "conflict-compact-tmp"));
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<WriteContext> compact_write_context,
                         compact_write_builder.Finish());
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> compact_writer,
                         FileStoreWrite::Create(std::move(compact_write_context)));
    ASSERT_OK(compact_writer->Compact(/*partition=*/{}, /*bucket=*/0,
                                      /*full_compaction=*/true));
    ASSERT_OK_AND_ASSIGN(
        std::vector<std::shared_ptr<CommitMessage>> compact_messages,
        compact_writer->PrepareCommit(/*wait_compaction=*/true, /*commit_identifier=*/0));
    ASSERT_EQ(1, compact_messages.size());
    std::shared_ptr<CommitMessageImpl> compact_message =
        std::dynamic_pointer_cast<CommitMessageImpl>(compact_messages[0]);
    ASSERT_NE(nullptr, compact_message);
    ASSERT_GE(compact_message->GetCompactIncrement().CompactBefore().size(), 2);
    ASSERT_FALSE(compact_message->GetCompactIncrement().CompactAfter().empty());

    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> append_batch,
                         MakeBatch(MakeRows(/*first_id=*/2, /*count=*/1, /*partition=*/"p0"),
                                   /*partitioned=*/false));
    ASSERT_OK(writer->Write(std::move(append_batch)));
    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> append_progress,
                         writer->PrepareCommitWithProgress(/*commit_identifier=*/2));

    const std::string conflict_snapshot_path =
        PathUtil::JoinPath(table_path_, "snapshot/snapshot-3");
    auto conflict_file_system =
        std::make_shared<SnapshotConflictFileSystem>(conflict_snapshot_path);
    CommitContextBuilder append_commit_builder(table_path_, commit_user_);
    ASSERT_OK_AND_ASSIGN(
        std::unique_ptr<CommitContext> append_commit_context,
        append_commit_builder.SetOptions(options_).WithFileSystem(conflict_file_system).Finish());
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreCommit> append_commit,
                         FileStoreCommit::Create(std::move(append_commit_context)));
    CommitContextBuilder compact_commit_builder(table_path_, "compaction_commit_user");
    ASSERT_OK_AND_ASSIGN(
        std::unique_ptr<CommitContext> compact_commit_context,
        compact_commit_builder.SetOptions(options_).WithFileSystem(conflict_file_system).Finish());
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreCommit> compact_commit,
                         FileStoreCommit::Create(std::move(compact_commit_context)));

    std::optional<Result<int64_t>> append_result;
    std::optional<Status> compact_status;
    std::thread append_thread([&]() {
        // CommitWithProgress probes the latest snapshot once for idempotency and once while
        // starting TryCommit. Synchronize only its destination check.
        conflict_file_system->SetPassthroughChecksForCurrentThread(/*checks=*/2);
        append_result.emplace(append_commit->CommitWithProgress(
            append_progress, /*commit_identifier=*/2, /*watermark=*/std::nullopt));
    });
    std::thread compact_thread([&]() {
        // A regular commit probes the latest snapshot once before its destination check.
        conflict_file_system->SetPassthroughChecksForCurrentThread(/*checks=*/1);
        compact_status.emplace(compact_commit->Commit(compact_messages, /*commit_identifier=*/0,
                                                      /*watermark=*/std::nullopt));
    });
    append_thread.join();
    compact_thread.join();

    ASSERT_TRUE(append_result.has_value());
    ASSERT_TRUE(compact_status.has_value());
    ASSERT_OK_AND_ASSIGN(int64_t append_snapshot_id, std::move(append_result.value()));
    ASSERT_OK(compact_status.value());
    ASSERT_TRUE(append_snapshot_id == 3 || append_snapshot_id == 4);
    ASSERT_EQ(2, conflict_file_system->ConflictSnapshotAttempts());

    ASSERT_OK_AND_ASSIGN(uint64_t append_attempts, append_commit->GetCommitMetrics()->GetCounter(
                                                       CommitMetrics::LAST_COMMIT_ATTEMPTS));
    ASSERT_OK_AND_ASSIGN(uint64_t compact_attempts, compact_commit->GetCommitMetrics()->GetCounter(
                                                        CommitMetrics::LAST_COMMIT_ATTEMPTS));
    std::vector<uint64_t> attempts = {append_attempts, compact_attempts};
    std::sort(attempts.begin(), attempts.end());
    ASSERT_EQ((std::vector<uint64_t>{1, 2}), attempts);

    ASSERT_OK_AND_ASSIGN(CoreOptions core_options, CoreOptions::FromMap(options_));
    SnapshotManager snapshot_manager(core_options.GetFileSystem(), table_path_);
    ASSERT_OK_AND_ASSIGN(Snapshot snapshot3, snapshot_manager.LoadSnapshot(/*snapshot_id=*/3));
    ASSERT_OK_AND_ASSIGN(Snapshot snapshot4, snapshot_manager.LoadSnapshot(/*snapshot_id=*/4));
    ASSERT_TRUE((snapshot3.GetCommitKind() == Snapshot::CommitKind::Append() &&
                 snapshot4.GetCommitKind() == Snapshot::CommitKind::Compact()) ||
                (snapshot3.GetCommitKind() == Snapshot::CommitKind::Compact() &&
                 snapshot4.GetCommitKind() == Snapshot::CommitKind::Append()));

    ASSERT_OK_AND_ASSIGN(RealtimeOffsetMap committed_offsets, ReadCommittedOffsets());
    ASSERT_EQ(1, committed_offsets.size());
    ASSERT_EQ(3, committed_offsets.at(RealtimePartitionBucket(/*partition=*/{}, /*bucket=*/0)));
    ASSERT_OK_AND_ASSIGN(std::vector<Row> rows, ReadRows());
    ASSERT_EQ(MakeRows(/*first_id=*/0, /*count=*/3, /*partition=*/"p0"), rows);

    ASSERT_OK_AND_ASSIGN(std::shared_ptr<Plan> plan,
                         CreatePlan(/*realtime_context=*/nullptr, /*predicate=*/nullptr));
    ASSERT_OK_AND_ASSIGN(DiskFileLayout layout, InspectDiskFiles(plan->Splits()));
    ASSERT_TRUE(layout.has_level0_file);
    ASSERT_TRUE(layout.has_high_level_file);
    ASSERT_OK(writer->Close());
    ASSERT_OK(compact_writer->Close());
}

TEST_F(RealtimeWriteInteTest, TestSealOnlySealsRealtimeSegment) {
    CreateTable(/*partition_keys=*/{});
    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
                         RealtimeContext::Create());
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
                         CreateRealtimeWriter(realtime_context));

    std::vector<Row> first_rows = MakeRows(/*first_id=*/0, /*count=*/3, /*partition=*/"p0");
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> first_batch,
                         MakeBatch(first_rows, /*partitioned=*/false));
    ASSERT_OK(writer->Write(std::move(first_batch)));
    ASSERT_OK_AND_ASSIGN(uint64_t memory_before_seal, GetRealtimeMemoryUsage(realtime_context));
    ASSERT_GT(memory_before_seal, 0);
    ASSERT_OK(writer->Seal());
    ASSERT_OK_AND_ASSIGN(uint64_t memory_after_seal, GetRealtimeMemoryUsage(realtime_context));
    ASSERT_EQ(memory_before_seal, memory_after_seal);
    ASSERT_OK_AND_ASSIGN(std::vector<Row> sealed_rows, ReadRows(realtime_context));
    ASSERT_EQ(first_rows, sealed_rows);
    ASSERT_OK_AND_ASSIGN(std::vector<Row> disk_rows_before_prepare, ReadRows());
    ASSERT_TRUE(disk_rows_before_prepare.empty());

    std::vector<Row> second_rows = MakeRows(/*first_id=*/3, /*count=*/2, /*partition=*/"p0");
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> second_batch,
                         MakeBatch(second_rows, /*partitioned=*/false));
    ASSERT_OK(writer->Write(std::move(second_batch)));
    std::vector<Row> expected_rows = first_rows;
    expected_rows.insert(expected_rows.end(), second_rows.begin(), second_rows.end());
    ASSERT_OK_AND_ASSIGN(std::vector<Row> rows_before_prepare, ReadRows(realtime_context));
    ASSERT_EQ(expected_rows, rows_before_prepare);

    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> progress,
                         writer->PrepareCommitWithProgress(/*commit_identifier=*/0));
    ASSERT_EQ(1, progress.size());
    ASSERT_EQ(OffsetRange(0, 5), progress[0].offset_range);
    ASSERT_OK_AND_ASSIGN(std::vector<Row> rows_after_prepare, ReadRows(realtime_context));
    ASSERT_EQ(expected_rows, rows_after_prepare);
    ASSERT_OK_AND_ASSIGN(int64_t snapshot_id, Commit(progress, /*commit_identifier=*/0));
    ASSERT_OK(writer->RefreshCommittedSnapshot(snapshot_id));
    ASSERT_OK_AND_ASSIGN(std::vector<Row> rows_after_refresh, ReadRows(realtime_context));
    ASSERT_EQ(expected_rows, rows_after_refresh);
    ASSERT_OK(writer->Close());
    realtime_context.reset();
    ASSERT_OK_AND_ASSIGN(std::vector<Row> actual_rows, ReadRows());
    ASSERT_EQ(expected_rows, actual_rows);
}

TEST_F(RealtimeWriteInteTest, TestSparseExternalOffsetsCommitAndRecover) {
    CreateTable(/*partition_keys=*/{});
    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
                         RealtimeContext::Create());
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
                         CreateRealtimeWriter(realtime_context));
    const std::vector<int64_t> offsets = {10, 20, 30};
    std::vector<Row> rows;
    for (int64_t offset : offsets) {
        ResetExternalOffset(/*partition=*/{}, /*bucket=*/0, offset);
        std::vector<Row> one_row = MakeRows(static_cast<int64_t>(rows.size()), /*count=*/1, "p0");
        ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> batch,
                             MakeBatch(one_row, /*partitioned=*/false));
        ASSERT_OK(writer->Write(std::move(batch)));
        rows.push_back(one_row.front());
    }

    ASSERT_OK_AND_ASSIGN(std::vector<Row> realtime_rows, ReadRows(realtime_context));
    ASSERT_EQ(rows, realtime_rows);
    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> commits,
                         writer->PrepareCommitWithProgress(/*commit_identifier=*/0));
    ASSERT_EQ(1, commits.size());
    ASSERT_EQ(OffsetRange(10, 31), commits[0].offset_range);
    ASSERT_OK_AND_ASSIGN(int64_t snapshot_id, Commit(commits, /*commit_identifier=*/0));
    ASSERT_OK(writer->RefreshCommittedSnapshot(snapshot_id));
    ASSERT_OK_AND_ASSIGN(RealtimeOffsetMap committed_offsets, ReadCommittedOffsets());
    ASSERT_EQ(31, committed_offsets.at(RealtimePartitionBucket(/*partition=*/{}, /*bucket=*/0)));
    ASSERT_OK_AND_ASSIGN(std::vector<Row> actual_rows, ReadRows(realtime_context));
    ASSERT_EQ(rows, actual_rows);
    ASSERT_OK(writer->Close());
}

TEST_F(RealtimeWriteInteTest, TestExplicitSnapshotIdRequiresExistingSnapshot) {
    CreateTable(/*partition_keys=*/{});
    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
                         RealtimeContext::Create());
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
                         CreateRealtimeWriter(realtime_context));

    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> batch,
                         MakeBatch({{1, "memory", "p0"}}, /*partitioned=*/false));
    ASSERT_OK(writer->Write(std::move(batch)));

    options_[Options::SCAN_SNAPSHOT_ID] = "1";
    ASSERT_NOK_WITH_MSG(CreatePlan(realtime_context, /*predicate=*/nullptr),
                        "There is currently no snapshot.");
    ASSERT_OK(PrepareAndClose(writer.get()));
}

TEST_F(RealtimeWriteInteTest, TestExplicitSnapshotIdWithSparseTailAndReclaimBoundary) {
    CreateTable(/*partition_keys=*/{});
    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
                         RealtimeContext::Create());
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
                         CreateRealtimeWriter(realtime_context));

    ResetExternalOffset(/*partition=*/{}, /*bucket=*/0, /*next_offset=*/10);
    const std::vector<Row> disk_rows = {{1, "disk", "p0"}};
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> disk_batch,
                         MakeBatch(disk_rows, /*partitioned=*/false));
    ASSERT_OK(writer->Write(std::move(disk_batch)));
    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> disk_progress,
                         writer->PrepareCommitWithProgress(/*commit_identifier=*/0));
    ASSERT_OK_AND_ASSIGN(int64_t disk_snapshot_id, Commit(disk_progress, /*commit_identifier=*/0));
    ASSERT_OK(writer->RefreshCommittedSnapshot(disk_snapshot_id));

    // Offset gaps are valid. The snapshot high-watermark is 11 while the next memory row starts at
    // 30, so a real-time scan must not require the two numeric offsets to be adjacent.
    ResetExternalOffset(/*partition=*/{}, /*bucket=*/0, /*next_offset=*/30);
    const std::vector<Row> memory_rows = {{2, "memory", "p0"}};
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> memory_batch,
                         MakeBatch(memory_rows, /*partitioned=*/false));
    ASSERT_OK(writer->Write(std::move(memory_batch)));

    options_[Options::SCAN_SNAPSHOT_ID] = std::to_string(disk_snapshot_id);
    ASSERT_OK_AND_ASSIGN(std::shared_ptr<Plan> plan,
                         CreatePlan(realtime_context, /*predicate=*/nullptr));
    ASSERT_EQ(disk_snapshot_id, plan->SnapshotId());
    ASSERT_OK_AND_ASSIGN(std::vector<Row> actual_rows, ReadRows(plan, realtime_context));
    ASSERT_EQ((std::vector<Row>{{1, "disk", "p0"}, {2, "memory", "p0"}}), actual_rows);

    options_.erase(Options::SCAN_SNAPSHOT_ID);
    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> memory_progress,
                         writer->PrepareCommitWithProgress(/*commit_identifier=*/1));
    ASSERT_OK_AND_ASSIGN(int64_t memory_snapshot_id,
                         Commit(memory_progress, /*commit_identifier=*/1));
    ASSERT_OK(writer->RefreshCommittedSnapshot(memory_snapshot_id));

    options_[Options::SCAN_SNAPSHOT_ID] = std::to_string(disk_snapshot_id);
    ASSERT_NOK_WITH_MSG(CreatePlan(realtime_context, /*predicate=*/nullptr),
                        "behind context committed offset");
    ASSERT_OK(writer->Close());
}

TEST_F(RealtimeWriteInteTest, TestPkRead) {
    options_[Options::REALTIME_STORE_STATS_MODE] = "full";
    CreatePkTable();
    auto saw_query_predicate = std::make_shared<std::atomic<bool>>(false);
    auto query_view = std::make_shared<std::weak_ptr<RealtimeReadView>>();
    auto factory =
        MakeDecoratingFactory<QueryTrackingRealtimeStore>(saw_query_predicate, query_view);
    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
                         RealtimeContext::Create(factory));
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
                         CreateRealtimeWriter(realtime_context));

    std::vector<Row> first_rows = {{1, "old", "p0"}, {2, "two", "p0"}, {1, "new-in-run", "p0"}};
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> first_batch,
                         MakeBatch(first_rows, /*partitioned=*/false, /*bucket=*/0,
                                   {RecordBatch::RowKind::INSERT, RecordBatch::RowKind::INSERT,
                                    RecordBatch::RowKind::UPDATE_AFTER}));
    ASSERT_OK(writer->Write(std::move(first_batch)));
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> update_batch,
                         MakeBatch({Row{1, "new", "p0"}}, /*partitioned=*/false, /*bucket=*/0,
                                   {RecordBatch::RowKind::UPDATE_AFTER}));
    ASSERT_OK(writer->Write(std::move(update_batch)));
    ASSERT_OK_AND_ASSIGN(std::vector<Row> memory_rows, ReadRows(realtime_context));
    ASSERT_EQ((std::vector<Row>{{1, "new", "p0"}, {2, "two", "p0"}}), memory_rows);

    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> progress,
                         writer->PrepareCommitWithProgress(/*commit_identifier=*/0));
    ASSERT_EQ(1, progress.size());
    ASSERT_OK(Commit(progress, /*commit_identifier=*/0));

    std::vector<Row> second_rows = {{1, "latest", "p0"}, {2, "gone", "p0"}, {3, "three", "p0"}};
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> second_batch,
                         MakeBatch(second_rows, /*partitioned=*/false, /*bucket=*/0,
                                   {RecordBatch::RowKind::UPDATE_AFTER,
                                    RecordBatch::RowKind::DELETE, RecordBatch::RowKind::INSERT}));
    ASSERT_OK(writer->Write(std::move(second_batch)));
    ASSERT_OK_AND_ASSIGN(std::vector<Row> union_rows, ReadRows(realtime_context));
    ASSERT_EQ((std::vector<Row>{{1, "latest", "p0"}, {3, "three", "p0"}}), union_rows);

    const std::string expected_payload = "new";
    std::shared_ptr<Predicate> predicate = PredicateBuilder::Equal(
        /*field_index=*/1, /*field_name=*/"payload", FieldType::STRING,
        Literal(FieldType::STRING, expected_payload.data(), expected_payload.size()));
    ASSERT_OK_AND_ASSIGN(std::shared_ptr<Plan> filtered_plan,
                         CreatePlan(realtime_context, predicate));
    ASSERT_OK_AND_ASSIGN(std::shared_ptr<arrow::ChunkedArray> filtered_result,
                         ReadPlan(filtered_plan, realtime_context, {"id", "payload", "pt"},
                                  predicate, /*enable_predicate_filter=*/true));
    ASSERT_EQ(nullptr, filtered_result);
    ASSERT_FALSE(saw_query_predicate->load(std::memory_order_acquire));

    std::shared_ptr<Predicate> primary_key_predicate = PredicateBuilder::Equal(
        /*field_index=*/0, /*field_name=*/"id", FieldType::BIGINT, Literal(int64_t{3}));
    ASSERT_OK_AND_ASSIGN(std::shared_ptr<Plan> primary_key_plan,
                         CreatePlan(realtime_context, primary_key_predicate));
    ASSERT_OK_AND_ASSIGN(std::vector<Row> primary_key_rows,
                         ReadRows(primary_key_plan, realtime_context, primary_key_predicate,
                                  /*enable_predicate_filter=*/true));
    ASSERT_EQ((std::vector<Row>{{3, "three", "p0"}}), primary_key_rows);
    ASSERT_TRUE(saw_query_predicate->load(std::memory_order_acquire));
    ASSERT_OK(PrepareAndClose(writer.get()));
    writer.reset();

    ASSERT_OK_AND_ASSIGN(std::shared_ptr<Plan> lifetime_plan,
                         CreatePlan(realtime_context, /*predicate=*/nullptr));
    ReadContextBuilder read_builder(table_path_);
    read_builder.SetOptions(options_)
        .SetReadFieldNames({"id", "payload", "pt"})
        .WithRealtimeContext(realtime_context)
        .WithMemoryPool(pool_);
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<ReadContext> read_context, read_builder.Finish());
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<TableRead> table_read,
                         TableRead::Create(std::move(read_context)));
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<BatchReader> reader,
                         table_read->CreateReader(lifetime_plan->Splits()));
    ASSERT_FALSE(query_view->expired());

    std::weak_ptr<RealtimeContext> weak_context = realtime_context;
    table_read.reset();
    lifetime_plan.reset();
    realtime_context.reset();
    ASSERT_TRUE(weak_context.expired());
    ASSERT_FALSE(query_view->expired());
    ASSERT_OK_AND_ASSIGN(BatchReader::ReadBatch read_batch, reader->NextBatch());
    ASSERT_FALSE(BatchReader::IsEofBatch(read_batch));
    reader->Close();
    reader.reset();
    ASSERT_TRUE(query_view->expired());
    ASSERT_OK_AND_ASSIGN(std::shared_ptr<arrow::Array> read_array,
                         ReadResultCollector::GetArray(std::move(read_batch)));
    ASSERT_NE(nullptr, read_array);
    read_array.reset();
}

TEST_F(RealtimeWriteInteTest, TestPkVector) {
    options_[Options::FILE_FORMAT] = "parquet";
    std::shared_ptr<arrow::DataType> vector_type =
        arrow::fixed_size_list(arrow::field("item", arrow::float32(), /*nullable=*/false), 3);
    fields_ = {arrow::field("id", arrow::int64()), arrow::field("embedding", vector_type)};
    schema_ = arrow::schema(fields_);
    CreatePkTable();

    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
                         RealtimeContext::Create());
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
                         CreateRealtimeWriter(realtime_context));
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> batch, MakeUnpartitionedBatchFromJson(R"([
                             [0, 2, [2.0, 2.0, 2.0]],
                             [1, 3, null],
                             [2, 1, [1.0, 1.0, 1.0]],
                             [3, 2, [3.0, 3.0, 3.0]]
                         ])"));
    ASSERT_OK(writer->Write(std::move(batch)));

    ASSERT_OK_AND_ASSIGN(std::shared_ptr<Plan> plan,
                         CreatePlan(realtime_context, /*predicate=*/nullptr));
    ReadPlanWithSchemaAndCheck(plan, realtime_context, schema_,
                               R"([[0, 1, [1.0, 1.0, 1.0]],
                                   [0, 2, [3.0, 3.0, 3.0]],
                                   [0, 3, null]])");
    ASSERT_OK(PrepareAndClose(writer.get()));
}

TEST_F(RealtimeWriteInteTest, TestPkRealtimeReadOptimizedScanUnsupported) {
    CreatePkTable();
    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
                         RealtimeContext::Create());
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
                         CreateRealtimeWriter(realtime_context));
    const std::vector<Row> rows = {{1, "one", "p0"}, {2, "two", "p0"}};
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> batch,
                         MakeBatch(rows, /*partitioned=*/false));
    ASSERT_OK(writer->Write(std::move(batch)));
    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> progress,
                         writer->PrepareCommitWithProgress(/*commit_identifier=*/0));
    ASSERT_OK_AND_ASSIGN(int64_t snapshot_id, Commit(progress, /*commit_identifier=*/0));
    ASSERT_OK(writer->RefreshCommittedSnapshot(snapshot_id));
    ASSERT_OK_AND_ASSIGN(std::vector<Row> disk_rows, ReadRows());
    ASSERT_EQ(rows, disk_rows);

    ScanContextBuilder scan_builder(table_path_ + "$ro");
    scan_builder.SetOptions(options_).WithRealtimeContext(realtime_context).WithMemoryPool(pool_);
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<ScanContext> scan_context, scan_builder.Finish());
    Result<std::unique_ptr<TableScan>> scan = TableScan::Create(std::move(scan_context));
    ASSERT_TRUE(scan.status().IsNotImplemented()) << scan.status().ToString();
    ASSERT_NE(std::string::npos, scan.status().ToString().find(
                                     "PK real-time union read does not support read-optimized"));
    ASSERT_OK(writer->Close());
}

TEST_F(RealtimeWriteInteTest, TestPkDeleteInsertAndPinnedReadsAcrossRefresh) {
    CreatePkTable();
    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
                         RealtimeContext::Create());
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
                         CreateRealtimeWriter(realtime_context));

    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> delete_batch,
                         MakeBatch({Row{1, "deleted", "p0"}}, /*partitioned=*/false, /*bucket=*/0,
                                   {RecordBatch::RowKind::DELETE}));
    ASSERT_OK(writer->Write(std::move(delete_batch)));
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> insert_batch,
                         MakeBatch({Row{1, "inserted", "p0"}}, /*partitioned=*/false, /*bucket=*/0,
                                   {RecordBatch::RowKind::INSERT}));
    ASSERT_OK(writer->Write(std::move(insert_batch)));
    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> progress,
                         writer->PrepareCommitWithProgress(/*commit_identifier=*/0));
    ASSERT_EQ(1, progress.size());

    ASSERT_OK_AND_ASSIGN(std::shared_ptr<Plan> pinned_plan,
                         CreatePlan(realtime_context, /*predicate=*/nullptr));
    ASSERT_OK_AND_ASSIGN(std::shared_ptr<Plan> reader_plan,
                         CreatePlan(realtime_context, /*predicate=*/nullptr));
    ReadContextBuilder read_builder(table_path_);
    read_builder.SetOptions(options_)
        .SetReadFieldNames({"id", "payload", "pt"})
        .WithRealtimeContext(realtime_context)
        .WithMemoryPool(pool_);
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<ReadContext> read_context, read_builder.Finish());
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<TableRead> table_read,
                         TableRead::Create(std::move(read_context)));
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<BatchReader> pinned_reader,
                         table_read->CreateReader(reader_plan->Splits()));

    ASSERT_OK_AND_ASSIGN(int64_t snapshot_id, Commit(progress, /*commit_identifier=*/0));
    ASSERT_OK(writer->RefreshCommittedSnapshot(snapshot_id));
    ASSERT_OK_AND_ASSIGN(std::vector<Row> plan_rows, ReadRows(pinned_plan, realtime_context));
    ASSERT_EQ((std::vector<Row>{{1, "inserted", "p0"}}), plan_rows);
    ASSERT_OK_AND_ASSIGN(std::shared_ptr<arrow::ChunkedArray> reader_rows,
                         ReadResultCollector::CollectResult(std::move(pinned_reader)));
    ASSERT_EQ(1, reader_rows->length());
    ASSERT_OK(writer->Close());
}

TEST_F(RealtimeWriteInteTest, TestPkMergeDiskSealedAndActive) {
    options_[Options::READ_BATCH_SIZE] = "2";
    CreatePkTable();
    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
                         RealtimeContext::Create());
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
                         CreateRealtimeWriter(realtime_context));

    const std::vector<std::vector<Row>> disk_batches = {
        {{1, "disk-1", "p0"}, {2, "disk-2", "p0"}, {3, "disk-3", "p0"}},
        {{10, "disk-10", "p0"}, {11, "disk-11", "p0"}},
    };
    int64_t commit_identifier = 0;
    for (const std::vector<Row>& disk_rows : disk_batches) {
        ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> batch,
                             MakeBatch(disk_rows, /*partitioned=*/false));
        ASSERT_OK(writer->Write(std::move(batch)));
        ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> progress,
                             writer->PrepareCommitWithProgress(commit_identifier));
        ASSERT_EQ(1, progress.size());
        ASSERT_EQ(1, NewFiles(progress).size());
        ASSERT_OK_AND_ASSIGN(int64_t snapshot_id, Commit(progress, commit_identifier));
        ASSERT_OK(writer->RefreshCommittedSnapshot(snapshot_id));
        ++commit_identifier;
    }

    ASSERT_OK_AND_ASSIGN(
        std::unique_ptr<RecordBatch> sealed_batch,
        MakeBatch({Row{1, "sealed-1", "p0"}, Row{2, "deleted-2", "p0"}, Row{4, "sealed-4", "p0"}},
                  /*partitioned=*/false, /*bucket=*/0,
                  {RecordBatch::RowKind::UPDATE_AFTER, RecordBatch::RowKind::DELETE,
                   RecordBatch::RowKind::INSERT}));
    ASSERT_OK(writer->Write(std::move(sealed_batch)));
    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> sealed_progress,
                         writer->PrepareCommitWithProgress(/*commit_identifier=*/2));
    ASSERT_EQ(1, sealed_progress.size());

    ASSERT_OK_AND_ASSIGN(
        std::unique_ptr<RecordBatch> active_batch,
        MakeBatch({Row{1, "active-1", "p0"}, Row{4, "deleted-4", "p0"}, Row{5, "active-5", "p0"}},
                  /*partitioned=*/false, /*bucket=*/0,
                  {RecordBatch::RowKind::UPDATE_AFTER, RecordBatch::RowKind::DELETE,
                   RecordBatch::RowKind::INSERT}));
    ASSERT_OK(writer->Write(std::move(active_batch)));

    ASSERT_OK_AND_ASSIGN(std::shared_ptr<Plan> plan,
                         CreatePlan(realtime_context, /*predicate=*/nullptr));
    ASSERT_OK_AND_ASSIGN(std::shared_ptr<arrow::ChunkedArray> result,
                         ReadPlan(plan, realtime_context, {"payload", "id"}, /*predicate=*/nullptr,
                                  /*enable_predicate_filter=*/false));
    ASSERT_NE(nullptr, result);
    ASSERT_GT(result->num_chunks(), 1);
    for (const std::shared_ptr<arrow::Array>& chunk : result->chunks()) {
        ASSERT_LE(chunk->length(), 2);
    }
    std::shared_ptr<arrow::DataType> result_type = arrow::struct_(
        {arrow::field("_VALUE_KIND", arrow::int8()), arrow::field("payload", arrow::utf8()),
         arrow::field("id", arrow::int64())});
    std::shared_ptr<arrow::Array> expected =
        arrow::ipc::internal::json::ArrayFromJSON(result_type, R"([
            [0, "active-1", 1],
            [0, "disk-3", 3],
            [0, "active-5", 5],
            [0, "disk-10", 10],
            [0, "disk-11", 11]
        ])")
            .ValueOrDie();
    ASSERT_TRUE(std::make_shared<arrow::ChunkedArray>(expected)->Equals(*result))
        << result->ToString();
    ASSERT_OK(PrepareAndClose(writer.get()));
}

TEST_F(RealtimeWriteInteTest, TestPkMergeAllDiskSplitsWithMemory) {
    options_[Options::SOURCE_SPLIT_OPEN_FILE_COST] = "1";
    options_[Options::SOURCE_SPLIT_TARGET_SIZE] = "1";
    CreatePkTable();
    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
                         RealtimeContext::Create());
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
                         CreateRealtimeWriter(realtime_context));

    const std::vector<std::vector<Row>> disk_batches = {
        {{1, "disk-1", "p0"}, {2, "disk-2", "p0"}},
        {{10, "disk-10", "p0"}, {11, "disk-11", "p0"}},
        {{20, "disk-20", "p0"}, {21, "disk-21", "p0"}},
    };
    for (int64_t commit_identifier = 0;
         commit_identifier < static_cast<int64_t>(disk_batches.size()); ++commit_identifier) {
        ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> batch,
                             MakeBatch(disk_batches[commit_identifier], /*partitioned=*/false));
        ASSERT_OK(writer->Write(std::move(batch)));
        ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> progress,
                             writer->PrepareCommitWithProgress(commit_identifier));
        ASSERT_OK_AND_ASSIGN(int64_t snapshot_id, Commit(progress, commit_identifier));
        ASSERT_OK(writer->RefreshCommittedSnapshot(snapshot_id));
    }

    ASSERT_OK_AND_ASSIGN(
        std::unique_ptr<RecordBatch> memory_batch,
        MakeBatch({Row{1, "memory-1", "p0"}, Row{10, "deleted-10", "p0"}},
                  /*partitioned=*/false, /*bucket=*/0,
                  {RecordBatch::RowKind::UPDATE_AFTER, RecordBatch::RowKind::DELETE}));
    ASSERT_OK(writer->Write(std::move(memory_batch)));

    ASSERT_OK_AND_ASSIGN(std::shared_ptr<Plan> plan,
                         CreatePlan(realtime_context, /*predicate=*/nullptr));
    ASSERT_EQ(1, plan->Splits().size());
    std::shared_ptr<RealtimeSplit> realtime_split =
        std::dynamic_pointer_cast<RealtimeSplit>(plan->Splits()[0]);
    ASSERT_NE(nullptr, realtime_split);
    ASSERT_EQ(3, realtime_split->DiskSplits().size());

    ASSERT_OK_AND_ASSIGN(std::vector<Row> actual_rows, ReadRows(plan, realtime_context));
    ASSERT_EQ((std::vector<Row>{{1, "memory-1", "p0"},
                                {2, "disk-2", "p0"},
                                {11, "disk-11", "p0"},
                                {20, "disk-20", "p0"},
                                {21, "disk-21", "p0"}}),
              actual_rows);
    ASSERT_OK(PrepareAndClose(writer.get()));
}

TEST_F(RealtimeWriteInteTest, TestPkNestedProjectionAcrossDiskAndMemory) {
    const std::shared_ptr<arrow::Field> projected_b = arrow::field("b", arrow::int64());
    fields_ = {
        arrow::field("id", arrow::int64()),
        arrow::field("payload", arrow::struct_({arrow::field("a", arrow::int64()), projected_b})),
        arrow::field("pt", arrow::utf8()),
    };
    schema_ = arrow::schema(fields_);
    CreatePkTable();
    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
                         RealtimeContext::Create());
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
                         CreateRealtimeWriter(realtime_context));
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RealtimeSchemaLayout> schema_layout,
                         RealtimeSchemaLayout::Create(RealtimeStoreMode::PRIMARY_KEY, schema_));
    const std::shared_ptr<arrow::Schema>& realtime_schema = schema_layout->InputSchema();
    auto make_batch = [&](const std::string& json) -> Result<std::unique_ptr<RecordBatch>> {
        PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr<arrow::Array> array,
                                          arrow::ipc::internal::json::ArrayFromJSON(
                                              arrow::struct_(realtime_schema->fields()), json));
        ArrowArray c_array;
        PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*array, &c_array));
        return RecordBatchBuilder(&c_array).SetBucket(0).Finish();
    };

    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> disk_batch,
                         make_batch(R"([[0, 1, [101, 1001], "p0"],
                                        [1, 2, [102, 1002], "p0"]])"));
    ASSERT_OK(writer->Write(std::move(disk_batch)));
    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> disk_progress,
                         writer->PrepareCommitWithProgress(/*commit_identifier=*/0));
    ASSERT_OK_AND_ASSIGN(int64_t snapshot_id, Commit(disk_progress, /*commit_identifier=*/0));
    ASSERT_OK(writer->RefreshCommittedSnapshot(snapshot_id));

    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> sealed_batch,
                         make_batch(R"([[2, 1, [201, 2001], "p0"],
                                        [3, 3, [203, 2003], "p0"]])"));
    ASSERT_OK(writer->Write(std::move(sealed_batch)));
    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> sealed_progress,
                         writer->PrepareCommitWithProgress(/*commit_identifier=*/1));
    ASSERT_EQ(1, sealed_progress.size());

    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> active_batch,
                         make_batch(R"([[4, 1, [301, 3001], "p0"],
                                        [5, 4, [304, null], "p0"]])"));
    ASSERT_OK(writer->Write(std::move(active_batch)));
    ASSERT_OK_AND_ASSIGN(std::shared_ptr<Plan> plan,
                         CreatePlan(realtime_context, /*predicate=*/nullptr));

    auto projected_schema = arrow::schema({
        arrow::field("payload", arrow::struct_({projected_b})),
        arrow::field("id", arrow::int64()),
    });
    auto c_schema = std::make_unique<ArrowSchema>();
    ASSERT_TRUE(arrow::ExportSchema(*projected_schema, c_schema.get()).ok());
    ReadContextBuilder read_builder(table_path_);
    read_builder.SetOptions(options_)
        .SetReadSchema(std::move(c_schema))
        .WithRealtimeContext(realtime_context)
        .WithMemoryPool(pool_);
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<ReadContext> read_context, read_builder.Finish());
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<TableRead> table_read,
                         TableRead::Create(std::move(read_context)));
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<BatchReader> reader,
                         table_read->CreateReader(plan->Splits()));
    ASSERT_OK_AND_ASSIGN(std::shared_ptr<arrow::ChunkedArray> actual,
                         ReadResultCollector::CollectResult(std::move(reader)));
    const std::shared_ptr<arrow::DataType> result_type = arrow::struct_({
        arrow::field("_VALUE_KIND", arrow::int8()),
        arrow::field("payload", arrow::struct_({projected_b})),
        arrow::field("id", arrow::int64()),
    });
    const std::shared_ptr<arrow::Array> expected =
        arrow::ipc::internal::json::ArrayFromJSON(result_type, R"([
            [0, [3001], 1],
            [0, [1002], 2],
            [0, [2003], 3],
            [0, [null], 4]
        ])")
            .ValueOrDie();
    ASSERT_TRUE(std::make_shared<arrow::ChunkedArray>(expected)->Equals(*actual))
        << actual->ToString();
    ASSERT_OK(PrepareAndClose(writer.get()));
}

TEST_F(RealtimeWriteInteTest, TestPkKeylessProjection) {
    CreatePkTable();
    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
                         RealtimeContext::Create());
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
                         CreateRealtimeWriter(realtime_context));

    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> disk_batch,
                         MakeBatch({Row{1, "disk", "p0"}}, /*partitioned=*/false));
    ASSERT_OK(writer->Write(std::move(disk_batch)));
    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> disk_progress,
                         writer->PrepareCommitWithProgress(/*commit_identifier=*/0));
    ASSERT_OK_AND_ASSIGN(int64_t snapshot_id, Commit(disk_progress, /*commit_identifier=*/0));
    ASSERT_OK(writer->RefreshCommittedSnapshot(snapshot_id));

    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> memory_batch,
                         MakeBatch({Row{1, "memory", "p0"}}, /*partitioned=*/false));
    ASSERT_OK(writer->Write(std::move(memory_batch)));
    ASSERT_OK_AND_ASSIGN(std::shared_ptr<Plan> plan,
                         CreatePlan(realtime_context, /*predicate=*/nullptr));
    ReadPlanWithSchemaAndCheck(plan, realtime_context,
                               arrow::schema({arrow::field("payload", arrow::utf8())}), R"([
        [0, "memory"]
    ])");
    ASSERT_OK(PrepareAndClose(writer.get()));
}

TEST_F(RealtimeWriteInteTest, TestCompositePkKeylessProjection) {
    CreatePkTable(/*partition_keys=*/{}, /*primary_keys=*/{"id", "payload"});
    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
                         RealtimeContext::Create());
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
                         CreateRealtimeWriter(realtime_context));

    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> disk_batch,
                         MakeBatch({Row{1, "key", "disk"}}, /*partitioned=*/false));
    ASSERT_OK(writer->Write(std::move(disk_batch)));
    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> disk_progress,
                         writer->PrepareCommitWithProgress(/*commit_identifier=*/0));
    ASSERT_OK_AND_ASSIGN(int64_t snapshot_id, Commit(disk_progress, /*commit_identifier=*/0));
    ASSERT_OK(writer->RefreshCommittedSnapshot(snapshot_id));

    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> memory_batch,
                         MakeBatch({Row{1, "key", "memory"}}, /*partitioned=*/false));
    ASSERT_OK(writer->Write(std::move(memory_batch)));
    ASSERT_OK_AND_ASSIGN(std::shared_ptr<Plan> plan,
                         CreatePlan(realtime_context, /*predicate=*/nullptr));
    ReadPlanWithSchemaAndCheck(plan, realtime_context,
                               arrow::schema({arrow::field("pt", arrow::utf8())}), R"([
        [0, "memory"]
    ])");
    ASSERT_OK(PrepareAndClose(writer.get()));
}

TEST_F(RealtimeWriteInteTest, TestPkCompositeMerge) {
    CreatePkTable(/*partition_keys=*/{}, /*primary_keys=*/{"id", "payload"});
    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
                         RealtimeContext::Create());
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
                         CreateRealtimeWriter(realtime_context));

    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> disk_batch,
                         MakeBatch({Row{1, "a", "disk-1a"}, Row{1, "b", "disk-1b"},
                                    Row{2, "a", "disk-2a"}, Row{3, "c", "disk-3c"}},
                                   /*partitioned=*/false));
    ASSERT_OK(writer->Write(std::move(disk_batch)));
    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> disk_progress,
                         writer->PrepareCommitWithProgress(/*commit_identifier=*/0));
    ASSERT_EQ(1, disk_progress.size());
    ASSERT_EQ(OffsetRange(0, 4), disk_progress[0].offset_range);
    ASSERT_EQ(1, NewFiles(disk_progress).size());
    ASSERT_OK_AND_ASSIGN(int64_t snapshot_id, Commit(disk_progress, /*commit_identifier=*/0));
    ASSERT_OK(writer->RefreshCommittedSnapshot(snapshot_id));

    ASSERT_OK_AND_ASSIGN(
        std::unique_ptr<RecordBatch> sealed_batch,
        MakeBatch({Row{1, "a", "sealed-1a"}, Row{1, "b", "deleted-1b"}, Row{2, "b", "sealed-2b"}},
                  /*partitioned=*/false, /*bucket=*/0,
                  {RecordBatch::RowKind::UPDATE_AFTER, RecordBatch::RowKind::DELETE,
                   RecordBatch::RowKind::INSERT}));
    ASSERT_OK(writer->Write(std::move(sealed_batch)));
    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> sealed_progress,
                         writer->PrepareCommitWithProgress(/*commit_identifier=*/1));
    ASSERT_EQ(1, sealed_progress.size());
    ASSERT_EQ(OffsetRange(4, 7), sealed_progress[0].offset_range);
    ASSERT_EQ(1, NewFiles(sealed_progress).size());

    ASSERT_OK_AND_ASSIGN(
        std::unique_ptr<RecordBatch> active_batch,
        MakeBatch({Row{1, "a", "active-1a"}, Row{1, "c", "active-1c"}, Row{2, "a", "active-2a"}},
                  /*partitioned=*/false, /*bucket=*/0,
                  {RecordBatch::RowKind::UPDATE_AFTER, RecordBatch::RowKind::INSERT,
                   RecordBatch::RowKind::UPDATE_AFTER}));
    ASSERT_OK(writer->Write(std::move(active_batch)));

    ASSERT_OK_AND_ASSIGN(std::shared_ptr<Plan> plan,
                         CreatePlan(realtime_context, /*predicate=*/nullptr));
    ASSERT_EQ(1, plan->Splits().size());
    std::shared_ptr<RealtimeSplit> split =
        std::dynamic_pointer_cast<RealtimeSplit>(plan->Splits()[0]);
    ASSERT_NE(nullptr, split);
    ASSERT_FALSE(split->DiskSplits().empty());
    ASSERT_EQ(4, split->CommittedEndOffset());
    ASSERT_EQ(10, split->MemoryEndOffset());
    ASSERT_OK_AND_ASSIGN(std::vector<Row> actual_rows, ReadRows(plan, realtime_context));
    ASSERT_EQ((std::vector<Row>{{1, "a", "active-1a"},
                                {1, "c", "active-1c"},
                                {2, "a", "active-2a"},
                                {2, "b", "sealed-2b"},
                                {3, "c", "disk-3c"}}),
              actual_rows);
    ASSERT_OK(PrepareAndClose(writer.get()));
}

TEST_F(RealtimeWriteInteTest, TestPkWriterHandoff) {
    CreatePkTable();
    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
                         RealtimeContext::Create());
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> first_writer,
                         CreateRealtimeWriter(realtime_context));
    const std::vector<Row> first_rows = {
        {0, "value-0", "p0"}, {1, "value-1", "p0"}, {2, "value-2", "p0"}};
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> first_batch,
                         MakeBatch(first_rows, /*partitioned=*/false));
    ASSERT_OK(first_writer->Write(std::move(first_batch)));
    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> first_progress,
                         first_writer->PrepareCommitWithProgress(/*commit_identifier=*/0));
    ASSERT_EQ(1, first_progress.size());
    ASSERT_EQ(OffsetRange(0, 3), first_progress[0].offset_range);
    ASSERT_EQ(1, NewFiles(first_progress).size());
    ASSERT_EQ(0, NewFiles(first_progress)[0]->min_sequence_number);
    ASSERT_EQ(2, NewFiles(first_progress)[0]->max_sequence_number);
    ASSERT_OK(first_writer->Close());

    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> second_writer,
                         CreateRealtimeWriter(realtime_context));
    const std::vector<Row> second_rows = {{0, "updated-0", "p0"}, {3, "value-3", "p0"}};
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> second_batch,
                         MakeBatch(second_rows, /*partitioned=*/false));
    ASSERT_OK(second_writer->Write(std::move(second_batch)));
    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> second_progress,
                         second_writer->PrepareCommitWithProgress(/*commit_identifier=*/1));
    ASSERT_EQ(1, second_progress.size());
    ASSERT_EQ(OffsetRange(3, 5), second_progress[0].offset_range);
    ASSERT_EQ(1, NewFiles(second_progress).size());
    ASSERT_EQ(3, NewFiles(second_progress)[0]->min_sequence_number);
    ASSERT_EQ(4, NewFiles(second_progress)[0]->max_sequence_number);

    first_progress.push_back(std::move(second_progress[0]));
    ASSERT_OK(Commit(first_progress, /*commit_identifier=*/1));
    ASSERT_OK_AND_ASSIGN(std::vector<Row> actual_rows, ReadRows(realtime_context));
    ASSERT_EQ((std::vector<Row>{{0, "updated-0", "p0"},
                                {1, "value-1", "p0"},
                                {2, "value-2", "p0"},
                                {3, "value-3", "p0"}}),
              actual_rows);
    ASSERT_OK(second_writer->Close());
}

TEST_F(RealtimeWriteInteTest, TestPkPartitionBucketRecovery) {
    options_[Options::BUCKET] = "2";
    CreatePkTable(/*partition_keys=*/{"pt"});
    const RealtimePartitionBucket p0b0({{"pt", "p0"}}, /*bucket=*/0);
    const RealtimePartitionBucket p1b1({{"pt", "p1"}}, /*bucket=*/1);

    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> first_context, RealtimeContext::Create());
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> first_writer,
                         CreateRealtimeWriter(first_context));
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> p0_first_batch,
                         MakeBatch({Row{0, "p0-zero", "p0"}, Row{1, "p0-one", "p0"}},
                                   /*partitioned=*/true, /*bucket=*/0));
    ASSERT_OK(first_writer->Write(std::move(p0_first_batch)));
    ASSERT_OK_AND_ASSIGN(
        std::unique_ptr<RecordBatch> p1_first_batch,
        MakeBatch({Row{10, "p1-ten", "p1"}, Row{11, "p1-eleven", "p1"}, Row{12, "p1-twelve", "p1"}},
                  /*partitioned=*/true, /*bucket=*/1));
    ASSERT_OK(first_writer->Write(std::move(p1_first_batch)));
    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> first_progress,
                         first_writer->PrepareCommitWithProgress(/*commit_identifier=*/0));
    ASSERT_EQ(2, first_progress.size());
    std::map<RealtimePartitionBucket, OffsetRange> first_ranges;
    std::map<RealtimePartitionBucket, std::pair<int64_t, int64_t>> first_sequences;
    for (const RealtimeCommitProgress& progress : first_progress) {
        first_ranges.emplace(progress.partition_bucket, progress.offset_range);
        std::shared_ptr<CommitMessageImpl> message =
            std::dynamic_pointer_cast<CommitMessageImpl>(progress.commit_message);
        ASSERT_NE(nullptr, message);
        const std::vector<std::shared_ptr<DataFileMeta>>& files =
            message->GetNewFilesIncrement().NewFiles();
        ASSERT_EQ(1, files.size());
        first_sequences.emplace(
            progress.partition_bucket,
            std::make_pair(files[0]->min_sequence_number, files[0]->max_sequence_number));
    }
    ASSERT_EQ(OffsetRange(0, 2), first_ranges.at(p0b0));
    ASSERT_EQ(OffsetRange(0, 3), first_ranges.at(p1b1));
    ASSERT_EQ((std::pair<int64_t, int64_t>(0, 1)), first_sequences.at(p0b0));
    ASSERT_EQ((std::pair<int64_t, int64_t>(0, 2)), first_sequences.at(p1b1));
    ASSERT_OK_AND_ASSIGN(int64_t first_snapshot_id,
                         Commit(first_progress, /*commit_identifier=*/0));
    ASSERT_OK(first_writer->RefreshCommittedSnapshot(first_snapshot_id));
    ASSERT_OK(first_writer->Close());
    first_writer.reset();
    first_context.reset();

    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> second_context,
                         RealtimeContext::Create());
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> second_writer,
                         CreateRealtimeWriter(second_context));
    ASSERT_OK_AND_ASSIGN(
        std::unique_ptr<RecordBatch> p0_second_batch,
        MakeBatch({Row{0, "p0-zero-new", "p0"}, Row{2, "p0-two", "p0"}},
                  /*partitioned=*/true, /*bucket=*/0,
                  {RecordBatch::RowKind::UPDATE_AFTER, RecordBatch::RowKind::INSERT}));
    ASSERT_OK(second_writer->Write(std::move(p0_second_batch)));
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> p1_second_batch,
                         MakeBatch({Row{10, "p1-ten-deleted", "p1"}, Row{13, "p1-thirteen", "p1"}},
                                   /*partitioned=*/true, /*bucket=*/1,
                                   {RecordBatch::RowKind::DELETE, RecordBatch::RowKind::INSERT}));
    ASSERT_OK(second_writer->Write(std::move(p1_second_batch)));
    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> second_progress,
                         second_writer->PrepareCommitWithProgress(/*commit_identifier=*/1));
    ASSERT_EQ(2, second_progress.size());
    std::map<RealtimePartitionBucket, OffsetRange> second_ranges;
    std::map<RealtimePartitionBucket, std::pair<int64_t, int64_t>> second_sequences;
    for (const RealtimeCommitProgress& progress : second_progress) {
        second_ranges.emplace(progress.partition_bucket, progress.offset_range);
        std::shared_ptr<CommitMessageImpl> message =
            std::dynamic_pointer_cast<CommitMessageImpl>(progress.commit_message);
        ASSERT_NE(nullptr, message);
        const std::vector<std::shared_ptr<DataFileMeta>>& files =
            message->GetNewFilesIncrement().NewFiles();
        ASSERT_EQ(1, files.size());
        second_sequences.emplace(
            progress.partition_bucket,
            std::make_pair(files[0]->min_sequence_number, files[0]->max_sequence_number));
    }
    ASSERT_EQ(OffsetRange(2, 4), second_ranges.at(p0b0));
    ASSERT_EQ(OffsetRange(3, 5), second_ranges.at(p1b1));
    ASSERT_EQ((std::pair<int64_t, int64_t>(2, 3)), second_sequences.at(p0b0));
    ASSERT_EQ((std::pair<int64_t, int64_t>(3, 4)), second_sequences.at(p1b1));
    ASSERT_OK_AND_ASSIGN(int64_t second_snapshot_id,
                         Commit(second_progress, /*commit_identifier=*/1));
    ASSERT_OK(second_writer->RefreshCommittedSnapshot(second_snapshot_id));

    ASSERT_OK_AND_ASSIGN(std::vector<Row> actual_rows, ReadRows(second_context));
    std::sort(actual_rows.begin(), actual_rows.end());
    ASSERT_EQ((std::vector<Row>{{0, "p0-zero-new", "p0"},
                                {1, "p0-one", "p0"},
                                {2, "p0-two", "p0"},
                                {11, "p1-eleven", "p1"},
                                {12, "p1-twelve", "p1"},
                                {13, "p1-thirteen", "p1"}}),
              actual_rows);
    ASSERT_OK(second_writer->Close());

    ASSERT_OK_AND_ASSIGN(RealtimeOffsetMap offsets, ReadCommittedOffsets());
    ASSERT_EQ(2, offsets.size());
    ASSERT_EQ(4, offsets.at(p0b0));
    ASSERT_EQ(5, offsets.at(p1b1));
}

TEST_F(RealtimeWriteInteTest, TestPkRecovery) {
    CreatePkTable();

    WriteContextBuilder seed_builder(table_path_, commit_user_);
    seed_builder.SetOptions(options_).WithStreamingMode(true);
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<WriteContext> seed_context, seed_builder.Finish());
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> seed_writer,
                         FileStoreWrite::Create(std::move(seed_context)));
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> seed_batch,
                         MakeLogicalBatch({Row{99, "seed", "p0"}}, /*bucket=*/0));
    ASSERT_OK(seed_writer->Write(std::move(seed_batch)));
    ASSERT_OK_AND_ASSIGN(std::vector<std::shared_ptr<CommitMessage>> seed_messages,
                         seed_writer->PrepareCommit(/*wait_compaction=*/false,
                                                    /*commit_identifier=*/0));
    CommitContextBuilder seed_commit_builder(table_path_, commit_user_);
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<CommitContext> seed_commit_context,
                         seed_commit_builder.SetOptions(options_).Finish());
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreCommit> seed_commit,
                         FileStoreCommit::Create(std::move(seed_commit_context)));
    ASSERT_OK(seed_commit->Commit(seed_messages, /*commit_identifier=*/0));
    ASSERT_OK(seed_writer->Close());
    const std::vector<Row> mutations = {
        {1, "one", "p0"}, {1, "one-new", "p0"}, {2, "deleted", "p0"}, {3, "three", "p0"}};
    const std::vector<RecordBatch::RowKind> mutation_kinds = {
        RecordBatch::RowKind::INSERT, RecordBatch::RowKind::UPDATE_AFTER,
        RecordBatch::RowKind::DELETE, RecordBatch::RowKind::INSERT};

    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> first_context, RealtimeContext::Create());
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> first_writer,
                         CreateRealtimeWriter(first_context));
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> batch,
                         MakeBatch(mutations, /*partitioned=*/false, /*bucket=*/0, mutation_kinds));
    ASSERT_OK(first_writer->Write(std::move(batch)));
    ASSERT_OK_AND_ASSIGN(std::vector<int64_t> memory_sequences, ReadPkSequences(first_context));
    ASSERT_EQ((std::vector<int64_t>{1, 2, 3, 4}), memory_sequences);
    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> progress,
                         first_writer->PrepareCommitWithProgress(/*commit_identifier=*/1));
    ASSERT_EQ(1, progress.size());
    ASSERT_EQ(OffsetRange(0, 4), progress[0].offset_range);
    ASSERT_EQ(1, NewFiles(progress).size());
    ASSERT_EQ(2, NewFiles(progress)[0]->min_sequence_number);
    ASSERT_EQ(memory_sequences.back(), NewFiles(progress)[0]->max_sequence_number);
    ASSERT_EQ(1, NewFiles(progress)[0]->delete_row_count);
    ASSERT_OK(Commit(progress, /*commit_identifier=*/1));
    ASSERT_OK(first_writer->Close());
    first_context.reset();
    ASSERT_OK_AND_ASSIGN(std::vector<Row> rows_after_replay, ReadRows());
    ASSERT_EQ((std::vector<Row>{{1, "one-new", "p0"}, {3, "three", "p0"}, {99, "seed", "p0"}}),
              rows_after_replay);

    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> second_context,
                         RealtimeContext::Create());
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> second_writer,
                         CreateRealtimeWriter(second_context));
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> restart_batch,
                         MakeBatch({Row{4, "four", "p0"}}, /*partitioned=*/false));
    ASSERT_OK(second_writer->Write(std::move(restart_batch)));
    ASSERT_OK_AND_ASSIGN(std::vector<int64_t> restart_sequences, ReadPkSequences(second_context));
    ASSERT_EQ((std::vector<int64_t>{5}), restart_sequences);
    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> restart_progress,
                         second_writer->PrepareCommitWithProgress(/*commit_identifier=*/2));
    ASSERT_EQ(1, restart_progress.size());
    ASSERT_EQ(OffsetRange(4, 5), restart_progress[0].offset_range);
    ASSERT_EQ(5, NewFiles(restart_progress)[0]->min_sequence_number);
    ASSERT_EQ(5, NewFiles(restart_progress)[0]->max_sequence_number);
    ASSERT_OK(second_writer->Close());
}

TEST_F(RealtimeWriteInteTest, TestPkCompaction) {
    options_[Options::NUM_SORTED_RUNS_COMPACTION_TRIGGER] = "1";
    CreatePkTable();
    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
                         RealtimeContext::Create());
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
                         CreateRealtimeWriter(realtime_context));

    int64_t latest_snapshot_id = -1;
    constexpr int64_t kCommitRoundsBeforeCompaction = 4;
    std::set<std::string> committed_file_names;
    for (int64_t round = 0; round < kCommitRoundsBeforeCompaction; ++round) {
        const bool delete_latest_live_row = round == kCommitRoundsBeforeCompaction - 1;
        ASSERT_OK_AND_ASSIGN(
            std::unique_ptr<RecordBatch> batch,
            MakeBatch(
                {Row{delete_latest_live_row ? round - 1 : round,
                     delete_latest_live_row ? "deleted" : "value-" + std::to_string(round), "p0"}},
                /*partitioned=*/false, /*bucket=*/0,
                delete_latest_live_row
                    ? std::vector<RecordBatch::RowKind>{RecordBatch::RowKind::DELETE}
                    : std::vector<RecordBatch::RowKind>{}));
        ASSERT_OK(writer->Write(std::move(batch)));
        ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> progress,
                             writer->PrepareCommitWithProgress(round));
        ASSERT_EQ(1, progress.size());
        std::shared_ptr<CommitMessageImpl> message =
            std::dynamic_pointer_cast<CommitMessageImpl>(progress[0].commit_message);
        ASSERT_NE(nullptr, message);
        ASSERT_TRUE(message->GetCompactIncrement().IsEmpty());
        ASSERT_EQ(1, NewFiles(progress).size());
        committed_file_names.insert(NewFiles(progress)[0]->file_name);
        ASSERT_OK_AND_ASSIGN(latest_snapshot_id, Commit(progress, round));
        ASSERT_OK(writer->RefreshCommittedSnapshot(latest_snapshot_id));
        ASSERT_OK_AND_ASSIGN(uint64_t memory_usage, GetRealtimeMemoryUsage(realtime_context));
        ASSERT_EQ(0, memory_usage);
    }
    WriteContextBuilder compact_builder(table_path_, commit_user_);
    compact_builder.SetOptions(options_).WithStreamingMode(true);
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<WriteContext> compact_context, compact_builder.Finish());
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> compact_writer,
                         FileStoreWrite::Create(std::move(compact_context)));
    ASSERT_OK(compact_writer->Compact(/*partition=*/{}, /*bucket=*/0,
                                      /*full_compaction=*/true));
    ASSERT_OK_AND_ASSIGN(
        std::vector<std::shared_ptr<CommitMessage>> compact_messages,
        compact_writer->PrepareCommit(/*wait_compaction=*/true, /*commit_identifier=*/4));
    ASSERT_EQ(1, compact_messages.size());
    std::shared_ptr<CommitMessageImpl> compact_message =
        std::dynamic_pointer_cast<CommitMessageImpl>(compact_messages[0]);
    ASSERT_NE(nullptr, compact_message);
    ASSERT_TRUE(compact_message->GetNewFilesIncrement().IsEmpty());
    ASSERT_EQ(kCommitRoundsBeforeCompaction,
              compact_message->GetCompactIncrement().CompactBefore().size());
    std::set<std::string> compacted_file_names;
    for (const std::shared_ptr<DataFileMeta>& file :
         compact_message->GetCompactIncrement().CompactBefore()) {
        compacted_file_names.insert(file->file_name);
    }
    ASSERT_EQ(committed_file_names, compacted_file_names);
    ASSERT_FALSE(compact_message->GetCompactIncrement().CompactAfter().empty());
    constexpr int64_t kHistoricalMaxSequenceNumber = kCommitRoundsBeforeCompaction - 1;
    int64_t compacted_live_max_sequence_number = -1;
    for (const std::shared_ptr<DataFileMeta>& file :
         compact_message->GetCompactIncrement().CompactAfter()) {
        compacted_live_max_sequence_number =
            std::max(compacted_live_max_sequence_number, file->max_sequence_number);
    }
    ASSERT_LT(compacted_live_max_sequence_number, kHistoricalMaxSequenceNumber);
    CommitContextBuilder commit_builder(table_path_, commit_user_);
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<CommitContext> commit_context,
                         commit_builder.SetOptions(options_).Finish());
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreCommit> commit,
                         FileStoreCommit::Create(std::move(commit_context)));
    ASSERT_OK(commit->Commit(compact_messages, /*commit_identifier=*/4));
    ASSERT_OK(compact_writer->Close());

    ASSERT_OK_AND_ASSIGN(CoreOptions options, CoreOptions::FromMap(options_));
    SnapshotManager snapshot_manager(options.GetFileSystem(), table_path_);
    ASSERT_OK_AND_ASSIGN(std::optional<Snapshot> compact_snapshot,
                         snapshot_manager.LatestSnapshot());
    ASSERT_TRUE(compact_snapshot);
    ASSERT_EQ(Snapshot::CommitKind::Compact(), compact_snapshot->GetCommitKind());
    ASSERT_OK_AND_ASSIGN(RealtimeOffsetMap offsets, ReadCommittedOffsets());
    ASSERT_EQ(4, offsets.at(RealtimePartitionBucket(/*partition=*/{}, /*bucket=*/0)));
    ASSERT_OK_AND_ASSIGN(std::vector<Row> compacted_rows, ReadRows());
    ASSERT_EQ((std::vector<Row>{{0, "value-0", "p0"}, {1, "value-1", "p0"}}), compacted_rows);
    ASSERT_OK(writer->Close());
    writer.reset();
    realtime_context.reset();

    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> fresh_context, RealtimeContext::Create());
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> fresh_writer,
                         CreateRealtimeWriter(fresh_context));
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> fresh_batch,
                         MakeBatch({Row{4, "value-4", "p0"}},
                                   /*partitioned=*/false));
    ASSERT_OK(fresh_writer->Write(std::move(fresh_batch)));
    ASSERT_OK_AND_ASSIGN(std::vector<int64_t> fresh_sequences, ReadPkSequences(fresh_context));
    ASSERT_EQ((std::vector<int64_t>{compacted_live_max_sequence_number + 1}), fresh_sequences);
    ASSERT_LT(fresh_sequences.front(), kHistoricalMaxSequenceNumber);
    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> fresh_progress,
                         fresh_writer->PrepareCommitWithProgress(/*commit_identifier=*/5));
    ASSERT_EQ(1, fresh_progress.size());
    ASSERT_EQ(OffsetRange(4, 5), fresh_progress[0].offset_range);
    ASSERT_EQ(compacted_live_max_sequence_number + 1,
              NewFiles(fresh_progress)[0]->min_sequence_number);
    ASSERT_EQ(compacted_live_max_sequence_number + 1,
              NewFiles(fresh_progress)[0]->max_sequence_number);
    ASSERT_OK_AND_ASSIGN(latest_snapshot_id, Commit(fresh_progress, /*commit_identifier=*/5));
    ASSERT_OK(fresh_writer->Close());

    ASSERT_OK_AND_ASSIGN(offsets, ReadCommittedOffsets());
    ASSERT_EQ(5, offsets.at(RealtimePartitionBucket(/*partition=*/{}, /*bucket=*/0)));
    ASSERT_OK_AND_ASSIGN(std::vector<Row> final_rows, ReadRows());
    ASSERT_EQ((std::vector<Row>{{0, "value-0", "p0"}, {1, "value-1", "p0"}, {4, "value-4", "p0"}}),
              final_rows);
}

TEST_F(RealtimeWriteInteTest, TestPkMultipleStoredBatchesMergeForQueryAndCommit) {
    CreatePkTable();
    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
                         RealtimeContext::Create());
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
                         CreateRealtimeWriter(realtime_context));
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> first_batch,
                         MakeBatch({Row{4, "four", "p0"}, Row{2, "two", "p0"}, Row{1, "one", "p0"}},
                                   /*partitioned=*/false));
    ASSERT_OK(writer->Write(std::move(first_batch)));
    ASSERT_OK_AND_ASSIGN(
        std::unique_ptr<RecordBatch> second_batch,
        MakeBatch({Row{3, "three", "p0"}, Row{2, "deleted", "p0"}, Row{1, "one-new", "p0"}},
                  /*partitioned=*/false, /*bucket=*/0,
                  {RecordBatch::RowKind::INSERT, RecordBatch::RowKind::DELETE,
                   RecordBatch::RowKind::UPDATE_AFTER}));
    ASSERT_OK(writer->Write(std::move(second_batch)));

    const std::vector<Row> expected = {{1, "one-new", "p0"}, {3, "three", "p0"}, {4, "four", "p0"}};
    ASSERT_OK_AND_ASSIGN(std::vector<Row> query_rows, ReadRows(realtime_context));
    ASSERT_EQ(expected, query_rows);

    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> progress,
                         writer->PrepareCommitWithProgress(/*commit_identifier=*/0));
    ASSERT_EQ(1, progress.size());
    ASSERT_EQ(OffsetRange(0, 6), progress[0].offset_range);
    ASSERT_OK(Commit(progress, /*commit_identifier=*/0));
    ASSERT_OK_AND_ASSIGN(std::vector<Row> rows, ReadRows());
    ASSERT_EQ(expected, rows);
    ASSERT_OK(writer->Close());
}

TEST_F(RealtimeWriteInteTest, TestRollingFilesPreserveProgress) {
    options_[Options::TARGET_FILE_ROW_NUM] = "10";
    CreateTable(/*partition_keys=*/{});
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer, CreateRealtimeWriter());

    std::vector<Row> expected_rows;
    constexpr int64_t kBatchCount = 3;
    constexpr int64_t kRowsPerBatch = 10;
    for (int64_t batch_index = 0; batch_index < kBatchCount; ++batch_index) {
        std::vector<Row> rows =
            MakeRows(batch_index * kRowsPerBatch, kRowsPerBatch, /*partition=*/"p0");
        ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> batch,
                             MakeBatch(rows, /*partitioned=*/false));
        ASSERT_OK(writer->Write(std::move(batch)));
        expected_rows.insert(expected_rows.end(), rows.begin(), rows.end());
    }

    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> commits,
                         writer->PrepareCommitWithProgress(/*commit_identifier=*/0));
    ASSERT_EQ(1, commits.size());
    ASSERT_EQ(OffsetRange(0, kBatchCount * kRowsPerBatch), commits[0].offset_range);
    std::shared_ptr<CommitMessageImpl> commit_message =
        std::dynamic_pointer_cast<CommitMessageImpl>(commits[0].commit_message);
    ASSERT_NE(nullptr, commit_message);
    ASSERT_EQ(3, commit_message->GetNewFilesIncrement().NewFiles().size());
    ASSERT_OK(Commit(commits, /*commit_identifier=*/0));
    ASSERT_OK(writer->Close());

    ASSERT_OK_AND_ASSIGN(std::vector<Row> actual_rows, ReadRows());
    ASSERT_EQ(expected_rows, actual_rows);
}

TEST_F(RealtimeWriteInteTest, TestAppendScanKeepsDiskSplitsIndependent) {
    options_[Options::TARGET_FILE_ROW_NUM] = "2";
    options_[Options::SOURCE_SPLIT_OPEN_FILE_COST] = "1";
    options_[Options::SOURCE_SPLIT_TARGET_SIZE] = "1";
    CreateTable(/*partition_keys=*/{});
    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
                         RealtimeContext::Create());
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
                         CreateRealtimeWriter(realtime_context));

    std::vector<Row> expected_rows;
    for (int64_t first_id = 0; first_id < 6; first_id += 2) {
        std::vector<Row> rows = MakeRows(first_id, /*count=*/2, /*partition=*/"p0");
        ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> batch,
                             MakeBatch(rows, /*partitioned=*/false));
        ASSERT_OK(writer->Write(std::move(batch)));
        expected_rows.insert(expected_rows.end(), rows.begin(), rows.end());
    }
    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> commits,
                         writer->PrepareCommitWithProgress(/*commit_identifier=*/0));
    ASSERT_OK_AND_ASSIGN(int64_t snapshot_id, Commit(commits, /*commit_identifier=*/0));
    ASSERT_OK(writer->RefreshCommittedSnapshot(snapshot_id));

    std::vector<Row> memory_rows = MakeRows(/*first_id=*/6, /*count=*/2, /*partition=*/"p0");
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> memory_batch,
                         MakeBatch(memory_rows, /*partitioned=*/false));
    ASSERT_OK(writer->Write(std::move(memory_batch)));
    expected_rows.insert(expected_rows.end(), memory_rows.begin(), memory_rows.end());

    ASSERT_OK_AND_ASSIGN(std::shared_ptr<Plan> plan,
                         CreatePlan(realtime_context, /*predicate=*/nullptr));
    ASSERT_EQ(3, plan->Splits().size());
    ASSERT_EQ(nullptr, std::dynamic_pointer_cast<RealtimeSplit>(plan->Splits()[0]));
    ASSERT_EQ(nullptr, std::dynamic_pointer_cast<RealtimeSplit>(plan->Splits()[1]));
    std::shared_ptr<RealtimeSplit> realtime_split =
        std::dynamic_pointer_cast<RealtimeSplit>(plan->Splits()[2]);
    ASSERT_NE(nullptr, realtime_split);
    ASSERT_EQ(1, realtime_split->DiskSplits().size());
    ASSERT_EQ(6, realtime_split->CommittedEndOffset());
    ASSERT_EQ(8, realtime_split->MemoryEndOffset());

    ASSERT_OK_AND_ASSIGN(std::vector<Row> actual_rows, ReadRows(plan, realtime_context));
    ASSERT_EQ(expected_rows, actual_rows);
    ASSERT_OK(PrepareAndClose(writer.get()));
}

TEST_F(RealtimeWriteInteTest, TestAppendRealtimeReadUsesEmbeddedAndExternalBitmapIndexes) {
    options_["file-index.bitmap.columns"] = "payload";
    options_[Options::FILE_INDEX_IN_MANIFEST_THRESHOLD] = "1MB";
    options_[Options::REALTIME_STORE_STATS_MODE] = "full";
    CreateTable(/*partition_keys=*/{});
    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
                         RealtimeContext::Create());

    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> embedded_writer,
                         CreateRealtimeWriter(realtime_context));
    ASSERT_OK_AND_ASSIGN(
        std::unique_ptr<RecordBatch> embedded_batch,
        MakeBatch(
            {{1, "embedded-one", "p0"}, {2, "embedded-match", "p0"}, {3, "embedded-three", "p0"}},
            /*partitioned=*/false));
    ASSERT_OK(embedded_writer->Write(std::move(embedded_batch)));
    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> embedded_progress,
                         embedded_writer->PrepareCommitWithProgress(/*commit_identifier=*/0));
    std::vector<std::shared_ptr<DataFileMeta>> embedded_files = NewFiles(embedded_progress);
    ASSERT_EQ(1, embedded_files.size());
    ASSERT_NE(nullptr, embedded_files[0]->embedded_index);
    ASSERT_TRUE(embedded_files[0]->extra_files.empty());
    ASSERT_OK_AND_ASSIGN(int64_t embedded_snapshot_id,
                         Commit(embedded_progress, /*commit_identifier=*/0));
    ASSERT_OK(embedded_writer->RefreshCommittedSnapshot(embedded_snapshot_id));
    ASSERT_OK(embedded_writer->Close());

    options_[Options::FILE_INDEX_IN_MANIFEST_THRESHOLD] = "1B";
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> external_writer,
                         CreateRealtimeWriter(realtime_context));
    ASSERT_OK_AND_ASSIGN(
        std::unique_ptr<RecordBatch> external_batch,
        MakeBatch(
            {{4, "external-four", "p0"}, {5, "external-match", "p0"}, {6, "external-six", "p0"}},
            /*partitioned=*/false));
    ASSERT_OK(external_writer->Write(std::move(external_batch)));
    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> external_progress,
                         external_writer->PrepareCommitWithProgress(/*commit_identifier=*/1));
    std::vector<std::shared_ptr<DataFileMeta>> external_files = NewFiles(external_progress);
    ASSERT_EQ(1, external_files.size());
    ASSERT_EQ(nullptr, external_files[0]->embedded_index);
    ASSERT_EQ(1, external_files[0]->extra_files.size());
    ASSERT_TRUE(external_files[0]->extra_files[0].has_value());
    std::string external_index_path =
        PathUtil::JoinPath(table_path_, "bucket-0/" + external_files[0]->extra_files[0].value());
    ASSERT_OK_AND_ASSIGN(bool external_index_exists,
                         dir_->GetFileSystem()->Exists(external_index_path));
    ASSERT_TRUE(external_index_exists);
    ASSERT_OK_AND_ASSIGN(int64_t external_snapshot_id,
                         Commit(external_progress, /*commit_identifier=*/1));
    ASSERT_OK(external_writer->RefreshCommittedSnapshot(external_snapshot_id));
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> memory_batch,
                         MakeBatch({{7, "memory-seven", "p0"}}, /*partitioned=*/false));
    ASSERT_OK(external_writer->Write(std::move(memory_batch)));

    const std::string embedded_match = "embedded-match";
    const std::string external_match = "external-match";
    ASSERT_OK_AND_ASSIGN(
        std::shared_ptr<Predicate> predicate,
        PredicateBuilder::Or(
            {PredicateBuilder::Equal(
                 /*field_index=*/1, /*field_name=*/"payload", FieldType::STRING,
                 Literal(FieldType::STRING, embedded_match.data(), embedded_match.size())),
             PredicateBuilder::Equal(
                 /*field_index=*/1, /*field_name=*/"payload", FieldType::STRING,
                 Literal(FieldType::STRING, external_match.data(), external_match.size()))}));
    ASSERT_OK_AND_ASSIGN(std::shared_ptr<Plan> plan, CreatePlan(realtime_context, predicate));
    bool has_realtime_split = false;
    for (const std::shared_ptr<Split>& split : plan->Splits()) {
        has_realtime_split |= std::dynamic_pointer_cast<RealtimeSplit>(split) != nullptr;
    }
    ASSERT_TRUE(has_realtime_split);
    // Residual filtering is disabled. Batch statistics remove the non-matching memory batch, while
    // the absence of the other disk rows verifies that both bitmap index storage forms are used.
    ASSERT_OK_AND_ASSIGN(std::vector<Row> actual_rows, ReadRows(plan, realtime_context, predicate,
                                                                /*enable_predicate_filter=*/false));
    std::sort(actual_rows.begin(), actual_rows.end());
    ASSERT_EQ((std::vector<Row>{{2, "embedded-match", "p0"}, {5, "external-match", "p0"}}),
              actual_rows);
    ASSERT_OK(PrepareAndClose(external_writer.get()));
}

TEST_F(RealtimeWriteInteTest, TestCommitOrdersPreparedOffsetRanges) {
    CreateTable(/*partition_keys=*/{});
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer, CreateRealtimeWriter());

    std::vector<Row> first_rows = MakeRows(/*first_id=*/0, /*count=*/3, /*partition=*/"p0");
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> first_batch,
                         MakeBatch(first_rows, /*partitioned=*/false));
    ASSERT_OK(writer->Write(std::move(first_batch)));
    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> commits,
                         writer->PrepareCommitWithProgress(/*commit_identifier=*/0));
    ASSERT_EQ(1, commits.size());
    ASSERT_EQ(OffsetRange(0, 3), commits[0].offset_range);

    std::vector<Row> second_rows = MakeRows(/*first_id=*/3, /*count=*/2, /*partition=*/"p0");
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> second_batch,
                         MakeBatch(second_rows, /*partitioned=*/false));
    ASSERT_OK(writer->Write(std::move(second_batch)));
    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> second_commits,
                         writer->PrepareCommitWithProgress(/*commit_identifier=*/1));
    ASSERT_EQ(1, second_commits.size());
    ASSERT_EQ(OffsetRange(3, 5), second_commits[0].offset_range);

    commits.push_back(std::move(second_commits[0]));
    std::reverse(commits.begin(), commits.end());
    ASSERT_OK(Commit(commits, /*commit_identifier=*/1));
    ASSERT_OK_AND_ASSIGN(RealtimeOffsetMap committed_offsets, ReadCommittedOffsets());
    ASSERT_EQ(5, committed_offsets.at(RealtimePartitionBucket(/*partition=*/{}, /*bucket=*/0)));
    ASSERT_OK(writer->Close());

    std::vector<Row> expected_rows = first_rows;
    expected_rows.insert(expected_rows.end(), second_rows.begin(), second_rows.end());
    ASSERT_OK_AND_ASSIGN(std::vector<Row> actual_rows, ReadRows());
    ASSERT_EQ(expected_rows, actual_rows);
}

TEST_F(RealtimeWriteInteTest, TestCommitWithProgressRetryReturnsLatestSnapshot) {
    CreateTable(/*partition_keys=*/{});
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer, CreateRealtimeWriter());
    std::vector<Row> expected_rows = MakeRows(/*first_id=*/0, /*count=*/3, /*partition=*/"p0");
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> batch,
                         MakeBatch(expected_rows, /*partitioned=*/false));
    ASSERT_OK(writer->Write(std::move(batch)));
    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> commits,
                         writer->PrepareCommitWithProgress(/*commit_identifier=*/0));

    ASSERT_OK_AND_ASSIGN(int64_t first_snapshot_id, Commit(commits, /*commit_identifier=*/0));
    ASSERT_OK_AND_ASSIGN(int64_t retry_snapshot_id, Commit(commits, /*commit_identifier=*/0));
    ASSERT_EQ(first_snapshot_id, retry_snapshot_id);

    ASSERT_OK(writer->RefreshCommittedSnapshot(first_snapshot_id));
    std::vector<Row> second_rows = MakeRows(/*first_id=*/3, /*count=*/2, /*partition=*/"p0");
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> second_batch,
                         MakeBatch(second_rows, /*partitioned=*/false));
    ASSERT_OK(writer->Write(std::move(second_batch)));
    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> second_commits,
                         writer->PrepareCommitWithProgress(/*commit_identifier=*/1));
    ASSERT_OK_AND_ASSIGN(int64_t second_snapshot_id,
                         Commit(second_commits, /*commit_identifier=*/1));

    ASSERT_OK_AND_ASSIGN(retry_snapshot_id, Commit(commits, /*commit_identifier=*/0));
    ASSERT_EQ(second_snapshot_id, retry_snapshot_id);
    ASSERT_NE(first_snapshot_id, retry_snapshot_id);
    ASSERT_OK_AND_ASSIGN(RealtimeOffsetMap committed_offsets, ReadCommittedOffsets());
    ASSERT_EQ(5, committed_offsets.at(RealtimePartitionBucket(/*partition=*/{}, /*bucket=*/0)));

    expected_rows.insert(expected_rows.end(), second_rows.begin(), second_rows.end());
    ASSERT_OK_AND_ASSIGN(std::vector<Row> actual_rows, ReadRows());
    ASSERT_EQ(expected_rows, actual_rows);
    ASSERT_OK(writer->Close());
}

TEST_F(RealtimeWriteInteTest, TestCommitWithProgressRejectsCoveredRangesFromAnotherUser) {
    CreateTable(/*partition_keys=*/{});
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer, CreateRealtimeWriter());
    std::vector<Row> expected_rows = MakeRows(/*first_id=*/0, /*count=*/3, /*partition=*/"p0");
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> batch,
                         MakeBatch(expected_rows, /*partitioned=*/false));
    ASSERT_OK(writer->Write(std::move(batch)));
    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> commits,
                         writer->PrepareCommitWithProgress(/*commit_identifier=*/0));
    ASSERT_OK(Commit(commits, /*commit_identifier=*/0));

    CommitContextBuilder builder(table_path_, "another_realtime_commit_user");
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<CommitContext> context,
                         builder.SetOptions(options_).Finish());
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreCommit> commit,
                         FileStoreCommit::Create(std::move(context)));
    ASSERT_NOK_WITH_MSG(commit->CommitWithProgress(commits, /*commit_identifier=*/0,
                                                   /*watermark=*/std::nullopt),
                        "another commit user or identifier");

    ASSERT_OK_AND_ASSIGN(std::vector<Row> actual_rows, ReadRows());
    ASSERT_EQ(expected_rows, actual_rows);
    ASSERT_OK(writer->Close());
}

TEST_F(RealtimeWriteInteTest, TestRealtimeWriteAcrossAppendCompaction) {
    options_[Options::TARGET_FILE_ROW_NUM] = "2";
    options_[Options::COMPACTION_MIN_FILE_NUM] = "2";
    CreateTable(/*partition_keys=*/{});
    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
                         RealtimeContext::Create());
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
                         CreateRealtimeWriter(realtime_context));

    std::vector<Row> first_rows;
    for (int64_t first_id = 0; first_id < 5; first_id += 2) {
        std::vector<Row> rows =
            MakeRows(first_id, std::min<int64_t>(2, 5 - first_id), /*partition=*/"p0");
        ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> batch,
                             MakeBatch(rows, /*partitioned=*/false));
        ASSERT_OK(writer->Write(std::move(batch)));
        first_rows.insert(first_rows.end(), rows.begin(), rows.end());
    }
    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> first_commits,
                         writer->PrepareCommitWithProgress(/*commit_identifier=*/0));
    ASSERT_EQ(1, first_commits.size());
    ASSERT_EQ(OffsetRange(0, 5), first_commits[0].offset_range);
    std::shared_ptr<CommitMessageImpl> first_commit_message =
        std::dynamic_pointer_cast<CommitMessageImpl>(first_commits[0].commit_message);
    ASSERT_NE(nullptr, first_commit_message);
    ASSERT_EQ(3, first_commit_message->GetNewFilesIncrement().NewFiles().size());
    ASSERT_OK_AND_ASSIGN(int64_t first_snapshot_id, Commit(first_commits, /*commit_identifier=*/0));
    ASSERT_OK(writer->RefreshCommittedSnapshot(first_snapshot_id));

    ASSERT_OK_AND_ASSIGN(Snapshot compact_snapshot, CompactAndCommit(/*partition=*/{}, /*bucket=*/0,
                                                                     /*commit_identifier=*/1));
    ASSERT_EQ(Snapshot::CommitKind::Compact(), compact_snapshot.GetCommitKind());
    ASSERT_OK_AND_ASSIGN(RealtimeOffsetMap compacted_offsets, ReadCommittedOffsets());
    ASSERT_EQ(5, compacted_offsets.at(RealtimePartitionBucket(/*partition=*/{}, /*bucket=*/0)));

    ASSERT_OK(writer->RefreshCommittedSnapshot(compact_snapshot.Id()));
    ASSERT_OK_AND_ASSIGN(std::vector<Row> rows_after_compaction, ReadRows(realtime_context));
    ASSERT_EQ(first_rows, rows_after_compaction);

    std::vector<Row> second_rows = MakeRows(/*first_id=*/5, /*count=*/2, /*partition=*/"p0");
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> second_batch,
                         MakeBatch(second_rows, /*partitioned=*/false));
    ASSERT_OK(writer->Write(std::move(second_batch)));
    std::vector<Row> expected_rows = first_rows;
    expected_rows.insert(expected_rows.end(), second_rows.begin(), second_rows.end());
    ASSERT_OK_AND_ASSIGN(std::vector<Row> rows_with_building_memory, ReadRows(realtime_context));
    ASSERT_EQ(expected_rows, rows_with_building_memory);

    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> second_commits,
                         writer->PrepareCommitWithProgress(/*commit_identifier=*/2));
    ASSERT_EQ(1, second_commits.size());
    ASSERT_EQ(OffsetRange(5, 7), second_commits[0].offset_range);
    ASSERT_OK_AND_ASSIGN(int64_t final_snapshot_id,
                         Commit(second_commits, /*commit_identifier=*/2));
    ASSERT_OK_AND_ASSIGN(RealtimeOffsetMap final_offsets, ReadCommittedOffsets());
    ASSERT_EQ(7, final_offsets.at(RealtimePartitionBucket(/*partition=*/{}, /*bucket=*/0)));
    ASSERT_OK_AND_ASSIGN(std::vector<Row> final_rows_before_refresh, ReadRows(realtime_context));
    ASSERT_EQ(expected_rows, final_rows_before_refresh);
    ASSERT_OK(writer->RefreshCommittedSnapshot(final_snapshot_id));
    ASSERT_OK_AND_ASSIGN(std::vector<Row> final_rows_after_refresh, ReadRows(realtime_context));
    ASSERT_EQ(expected_rows, final_rows_after_refresh);
    ASSERT_OK(writer->Close());
}

TEST_F(RealtimeWriteInteTest, TestPkDvExternalBitmapCanEliminateEntireDiskFile) {
    options_[Options::FILE_FORMAT] = "parquet";
    options_[Options::DELETION_VECTORS_ENABLED] = "true";
    options_[Options::FILE_INDEX_READ_ENABLED] = "true";
    options_["file-index.bitmap.columns"] = "payload";
    options_[Options::FILE_INDEX_IN_MANIFEST_THRESHOLD] = "1B";
    options_[Options::REALTIME_STORE_STATS_MODE] = "full";
    CreatePkTable();
    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
                         RealtimeContext::Create());
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
                         CreateRealtimeWriter(realtime_context));

    // "middle" is inside the file's min/max range but absent from its bitmap index. Keeping the
    // index external makes scan planning retain the file and lets the merge reader eliminate it.
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> disk_batch,
                         MakeBatch({{1, "aaa", "p0"}, {2, "zzz", "p0"}}, /*partitioned=*/false));
    ASSERT_OK(writer->Write(std::move(disk_batch)));
    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> disk_progress,
                         writer->PrepareCommitWithProgress(/*commit_identifier=*/0));
    ASSERT_OK_AND_ASSIGN(int64_t disk_snapshot_id, Commit(disk_progress, /*commit_identifier=*/0));
    ASSERT_OK(writer->RefreshCommittedSnapshot(disk_snapshot_id));

    ASSERT_OK_AND_ASSIGN(Snapshot compact_snapshot, CompactAndCommit(/*partition=*/{}, /*bucket=*/0,
                                                                     /*commit_identifier=*/1));
    ASSERT_OK(writer->RefreshCommittedSnapshot(compact_snapshot.Id()));

    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> memory_batch,
                         MakeBatch({{3, "middle", "p0"}}, /*partitioned=*/false));
    ASSERT_OK(writer->Write(std::move(memory_batch)));

    const std::string middle = "middle";
    const std::shared_ptr<Predicate> predicate = PredicateBuilder::Equal(
        /*field_index=*/1, /*field_name=*/"payload", FieldType::STRING,
        Literal(FieldType::STRING, middle.data(), middle.size()));
    ASSERT_OK_AND_ASSIGN(std::shared_ptr<Plan> plan, CreatePlan(realtime_context, predicate));
    ASSERT_EQ(1, plan->Splits().size());
    std::shared_ptr<RealtimeSplit> realtime_split =
        std::dynamic_pointer_cast<RealtimeSplit>(plan->Splits()[0]);
    ASSERT_NE(nullptr, realtime_split);

    size_t external_high_level_file_count = 0;
    for (const std::shared_ptr<Split>& split : realtime_split->DiskSplits()) {
        std::shared_ptr<DataSplitImpl> data_split = std::dynamic_pointer_cast<DataSplitImpl>(split);
        ASSERT_NE(nullptr, data_split);
        for (const std::shared_ptr<DataFileMeta>& file : data_split->DataFiles()) {
            if (file->level > 0 && file->embedded_index == nullptr && !file->extra_files.empty() &&
                file->extra_files[0].has_value()) {
                ++external_high_level_file_count;
            }
        }
    }
    ASSERT_EQ(1, external_high_level_file_count);

    // The external bitmap removes the only disk reader. The in-memory matching row must still be
    // returned, and constructing the empty disk run must not rely on positional file/reader pairs.
    ASSERT_OK_AND_ASSIGN(std::vector<Row> actual_rows, ReadRows(plan, realtime_context, predicate,
                                                                /*enable_predicate_filter=*/false));
    ASSERT_EQ((std::vector<Row>{{3, "middle", "p0"}}), actual_rows);
    ASSERT_OK(PrepareAndClose(writer.get()));
}

TEST_F(RealtimeWriteInteTest, TestPkDvRealtimeReadUsesEmbeddedAndExternalBitmapIndexes) {
    options_[Options::FILE_FORMAT] = "parquet";
    options_[Options::FILE_COMPRESSION] = "none";
    options_[Options::DELETION_VECTORS_ENABLED] = "true";
    options_["file-index.bitmap.columns"] = "payload";
    options_[Options::FILE_INDEX_IN_MANIFEST_THRESHOLD] = "1MB";
    options_[Options::REALTIME_STORE_STATS_MODE] = "full";
    CreatePkTable();
    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
                         RealtimeContext::Create());
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> embedded_writer,
                         CreateRealtimeWriter(realtime_context));

    // Keep the max-level file substantially larger than the later L0 files. Otherwise the
    // size-ratio picker may rewrite all levels together instead of producing a deletion vector.
    // Put the padding in a non-indexed column so the data file is much larger than both L0 files
    // without making the embedded bitmap index itself large.
    const std::string first_padding(256 * 1024, 'X');
    const std::string third_padding(256 * 1024, 'Y');
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> base_batch,
                         MakeBatch({{1, "high-one", first_padding},
                                    {2, "high-match", "p0"},
                                    {3, "high-old-three", third_padding}},
                                   /*partitioned=*/false));
    ASSERT_OK(embedded_writer->Write(std::move(base_batch)));
    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> base_progress,
                         embedded_writer->PrepareCommitWithProgress(/*commit_identifier=*/0));
    std::vector<std::shared_ptr<DataFileMeta>> base_files = NewFiles(base_progress);
    ASSERT_EQ(1, base_files.size());
    ASSERT_NE(nullptr, base_files[0]->embedded_index);
    ASSERT_TRUE(base_files[0]->extra_files.empty());
    ASSERT_OK_AND_ASSIGN(int64_t base_snapshot_id, Commit(base_progress, /*commit_identifier=*/0));
    ASSERT_OK(embedded_writer->RefreshCommittedSnapshot(base_snapshot_id));

    ASSERT_OK_AND_ASSIGN(Snapshot full_compact_snapshot,
                         CompactAndCommit(/*partition=*/{}, /*bucket=*/0,
                                          /*commit_identifier=*/1));
    ASSERT_OK(embedded_writer->RefreshCommittedSnapshot(full_compact_snapshot.Id()));
    ASSERT_OK(embedded_writer->Close());

    options_[Options::FILE_INDEX_IN_MANIFEST_THRESHOLD] = "1B";
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> external_writer,
                         CreateRealtimeWriter(realtime_context));
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> external_batch,
                         MakeBatch({{3, "external-current-three", "p0"},
                                    {4, "external-four", "p0"},
                                    {5, "external-match", "p0"}},
                                   /*partitioned=*/false, /*bucket=*/0,
                                   {RecordBatch::RowKind::UPDATE_AFTER,
                                    RecordBatch::RowKind::INSERT, RecordBatch::RowKind::INSERT}));
    ASSERT_OK(external_writer->Write(std::move(external_batch)));
    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> external_progress,
                         external_writer->PrepareCommitWithProgress(/*commit_identifier=*/2));
    std::vector<std::shared_ptr<DataFileMeta>> external_files = NewFiles(external_progress);
    ASSERT_EQ(1, external_files.size());
    ASSERT_EQ(0, external_files[0]->level);
    ASSERT_EQ(nullptr, external_files[0]->embedded_index);
    ASSERT_EQ(1, external_files[0]->extra_files.size());
    ASSERT_TRUE(external_files[0]->extra_files[0].has_value());
    std::string external_index_path =
        PathUtil::JoinPath(table_path_, "bucket-0/" + external_files[0]->extra_files[0].value());
    ASSERT_OK_AND_ASSIGN(bool external_index_exists,
                         dir_->GetFileSystem()->Exists(external_index_path));
    ASSERT_TRUE(external_index_exists);
    ASSERT_OK_AND_ASSIGN(int64_t external_snapshot_id,
                         Commit(external_progress, /*commit_identifier=*/2));
    ASSERT_OK(external_writer->RefreshCommittedSnapshot(external_snapshot_id));

    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> second_external_batch,
                         MakeBatch({{6, "external-six", "p0"}}, /*partitioned=*/false));
    ASSERT_OK(external_writer->Write(std::move(second_external_batch)));
    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> second_external_progress,
                         external_writer->PrepareCommitWithProgress(/*commit_identifier=*/3));
    ASSERT_OK_AND_ASSIGN(int64_t second_external_snapshot_id,
                         Commit(second_external_progress, /*commit_identifier=*/3));
    ASSERT_OK(external_writer->RefreshCommittedSnapshot(second_external_snapshot_id));

    ASSERT_OK_AND_ASSIGN(Snapshot dv_compact_snapshot,
                         CompactAndCommit(/*partition=*/{}, /*bucket=*/0,
                                          /*commit_identifier=*/4,
                                          /*full_compaction=*/false));
    ASSERT_OK(external_writer->RefreshCommittedSnapshot(dv_compact_snapshot.Id()));

    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> memory_batch,
                         MakeBatch({{7, "memory-seven", "p0"}}, /*partitioned=*/false));
    ASSERT_OK(external_writer->Write(std::move(memory_batch)));

    const std::string high_match = "high-match";
    const std::string external_match = "external-match";
    ASSERT_OK_AND_ASSIGN(
        std::shared_ptr<Predicate> predicate,
        PredicateBuilder::Or(
            {PredicateBuilder::Equal(
                 /*field_index=*/1, /*field_name=*/"payload", FieldType::STRING,
                 Literal(FieldType::STRING, high_match.data(), high_match.size())),
             PredicateBuilder::Equal(
                 /*field_index=*/1, /*field_name=*/"payload", FieldType::STRING,
                 Literal(FieldType::STRING, external_match.data(), external_match.size()))}));
    ASSERT_OK_AND_ASSIGN(std::shared_ptr<Plan> plan, CreatePlan(realtime_context, predicate));
    ASSERT_EQ(1, plan->Splits().size());
    std::shared_ptr<RealtimeSplit> realtime_split =
        std::dynamic_pointer_cast<RealtimeSplit>(plan->Splits()[0]);
    ASSERT_NE(nullptr, realtime_split);
    ASSERT_OK_AND_ASSIGN(DiskFileLayout layout, InspectDiskFiles(realtime_split->DiskSplits()));
    ASSERT_TRUE(layout.has_high_level_file);
    ASSERT_TRUE(layout.has_high_level_deletion_vector);
    bool has_embedded_high_level_index_with_dv = false;
    bool has_external_high_level_index = false;
    for (const std::shared_ptr<Split>& split : realtime_split->DiskSplits()) {
        std::shared_ptr<DataSplitImpl> data_split = std::dynamic_pointer_cast<DataSplitImpl>(split);
        ASSERT_NE(nullptr, data_split);
        const std::vector<std::shared_ptr<DataFileMeta>>& files = data_split->DataFiles();
        const std::vector<std::optional<DeletionFile>>& deletion_files =
            data_split->DeletionFiles();
        for (size_t i = 0; i < files.size(); ++i) {
            has_embedded_high_level_index_with_dv |=
                files[i]->level > 0 && files[i]->embedded_index != nullptr &&
                !deletion_files.empty() && deletion_files[i].has_value();
            if (files[i]->level > 0 && files[i]->embedded_index == nullptr &&
                !files[i]->extra_files.empty() && files[i]->extra_files[0].has_value()) {
                has_external_high_level_index = true;
                std::string index_path =
                    PathUtil::JoinPath(data_split->BucketPath(), files[i]->extra_files[0].value());
                ASSERT_OK_AND_ASSIGN(bool index_exists, dir_->GetFileSystem()->Exists(index_path));
                ASSERT_TRUE(index_exists);
            }
        }
    }
    ASSERT_TRUE(has_embedded_high_level_index_with_dv);
    ASSERT_TRUE(has_external_high_level_index);

    // Only primary-key predicates are sent to L0 and memory in PK MOR reads. The non-key predicate
    // still reaches both high-level bitmap indexes, while the unfiltered memory row remains.
    ASSERT_OK_AND_ASSIGN(std::vector<Row> actual_rows, ReadRows(plan, realtime_context, predicate,
                                                                /*enable_predicate_filter=*/false));
    std::sort(actual_rows.begin(), actual_rows.end());
    ASSERT_EQ((std::vector<Row>{
                  {2, "high-match", "p0"}, {5, "external-match", "p0"}, {7, "memory-seven", "p0"}}),
              actual_rows);
    ASSERT_OK(PrepareAndClose(external_writer.get()));
}

TEST_F(RealtimeWriteInteTest, TestPkDvPredicateAcrossHighLevelLevel0AndMemory) {
    options_[Options::FILE_FORMAT] = "parquet";
    options_[Options::DELETION_VECTORS_ENABLED] = "true";
    options_[Options::REALTIME_STORE_STATS_MODE] = "full";
    CreatePkTable();
    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
                         RealtimeContext::Create());
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
                         CreateRealtimeWriter(realtime_context));

    const std::string matching_payload = "match-" + std::string(2048, 'X');
    const std::vector<Row> base_rows = {{1, matching_payload, "p0"},
                                        {2, matching_payload, "p0"},
                                        {3, matching_payload, "p0"},
                                        {4, "base-four", "p0"},
                                        {5, "base-five", "p0"}};
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> base_batch,
                         MakeBatch(base_rows, /*partitioned=*/false));
    ASSERT_OK(writer->Write(std::move(base_batch)));
    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> base_progress,
                         writer->PrepareCommitWithProgress(/*commit_identifier=*/0));
    ASSERT_OK_AND_ASSIGN(int64_t base_snapshot_id, Commit(base_progress, /*commit_identifier=*/0));
    ASSERT_OK(writer->RefreshCommittedSnapshot(base_snapshot_id));

    ASSERT_OK_AND_ASSIGN(Snapshot full_compact_snapshot,
                         CompactAndCommit(/*partition=*/{}, /*bucket=*/0,
                                          /*commit_identifier=*/1));
    ASSERT_OK(writer->RefreshCommittedSnapshot(full_compact_snapshot.Id()));
    ASSERT_OK_AND_ASSIGN(std::vector<Row> rows_after_full_compaction, ReadRows(realtime_context));
    ASSERT_EQ(base_rows, rows_after_full_compaction);

    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> dv_update_batch,
                         MakeBatch({Row{5, "dv-current-five", "p0"}}, /*partitioned=*/false,
                                   /*bucket=*/0, {RecordBatch::RowKind::UPDATE_AFTER}));
    ASSERT_OK(writer->Write(std::move(dv_update_batch)));
    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> dv_update_progress,
                         writer->PrepareCommitWithProgress(/*commit_identifier=*/2));
    ASSERT_OK_AND_ASSIGN(int64_t dv_update_snapshot_id,
                         Commit(dv_update_progress, /*commit_identifier=*/2));
    ASSERT_OK(writer->RefreshCommittedSnapshot(dv_update_snapshot_id));

    ASSERT_OK_AND_ASSIGN(Snapshot dv_compact_snapshot,
                         CompactAndCommit(/*partition=*/{}, /*bucket=*/0,
                                          /*commit_identifier=*/3,
                                          /*full_compaction=*/false));
    ASSERT_OK(writer->RefreshCommittedSnapshot(dv_compact_snapshot.Id()));
    const std::vector<Row> expected_high_level_rows = {{1, matching_payload, "p0"},
                                                       {2, matching_payload, "p0"},
                                                       {3, matching_payload, "p0"},
                                                       {4, "base-four", "p0"},
                                                       {5, "dv-current-five", "p0"}};
    ASSERT_OK_AND_ASSIGN(std::vector<Row> high_level_rows, ReadRows(realtime_context));
    ASSERT_EQ(expected_high_level_rows, high_level_rows);

    ASSERT_OK_AND_ASSIGN(
        std::unique_ptr<RecordBatch> level0_batch,
        MakeBatch({Row{1, "level0-current-one", "p0"}, Row{2, "deleted-two", "p0"}},
                  /*partitioned=*/false, /*bucket=*/0,
                  {RecordBatch::RowKind::UPDATE_AFTER, RecordBatch::RowKind::DELETE}));
    ASSERT_OK(writer->Write(std::move(level0_batch)));
    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> level0_progress,
                         writer->PrepareCommitWithProgress(/*commit_identifier=*/4));
    ASSERT_OK_AND_ASSIGN(int64_t level0_snapshot_id,
                         Commit(level0_progress, /*commit_identifier=*/4));
    ASSERT_OK(writer->RefreshCommittedSnapshot(level0_snapshot_id));
    const std::vector<Row> expected_level0_rows = {{1, "level0-current-one", "p0"},
                                                   {3, matching_payload, "p0"},
                                                   {4, "base-four", "p0"},
                                                   {5, "dv-current-five", "p0"}};
    ASSERT_OK_AND_ASSIGN(std::vector<Row> level0_rows, ReadRows(realtime_context));
    ASSERT_EQ(expected_level0_rows, level0_rows);

    ASSERT_OK_AND_ASSIGN(
        std::unique_ptr<RecordBatch> memory_batch,
        MakeBatch({Row{3, "memory-current-three", "p0"}, Row{7, matching_payload, "p0"}},
                  /*partitioned=*/false, /*bucket=*/0,
                  {RecordBatch::RowKind::UPDATE_AFTER, RecordBatch::RowKind::INSERT}));
    ASSERT_OK(writer->Write(std::move(memory_batch)));
    const std::vector<Row> expected_memory_rows = {{1, "level0-current-one", "p0"},
                                                   {3, "memory-current-three", "p0"},
                                                   {4, "base-four", "p0"},
                                                   {5, "dv-current-five", "p0"},
                                                   {7, matching_payload, "p0"}};
    ASSERT_OK_AND_ASSIGN(std::vector<Row> memory_rows, ReadRows(realtime_context));
    ASSERT_EQ(expected_memory_rows, memory_rows);

    std::shared_ptr<Predicate> predicate = PredicateBuilder::Equal(
        /*field_index=*/1, /*field_name=*/"payload", FieldType::STRING,
        Literal(FieldType::STRING, matching_payload.data(), matching_payload.size()));
    options_[Options::SCAN_SNAPSHOT_ID] = std::to_string(level0_snapshot_id);
    ASSERT_OK_AND_ASSIGN(std::shared_ptr<Plan> plan, CreatePlan(realtime_context, predicate));
    ASSERT_EQ(level0_snapshot_id, plan->SnapshotId());
    ASSERT_EQ(1, plan->Splits().size());
    std::shared_ptr<RealtimeSplit> realtime_split =
        std::dynamic_pointer_cast<RealtimeSplit>(plan->Splits()[0]);
    ASSERT_NE(nullptr, realtime_split);
    ASSERT_GT(realtime_split->MemoryEndOffset(), realtime_split->CommittedEndOffset());

    ASSERT_OK_AND_ASSIGN(DiskFileLayout layout, InspectDiskFiles(realtime_split->DiskSplits()));
    ASSERT_TRUE(layout.has_level0_file);
    ASSERT_TRUE(layout.has_high_level_file);
    ASSERT_TRUE(layout.has_high_level_deletion_vector);

    // Predicate pushdown may return false-positive candidates, but a newer unfiltered L0 or
    // memory record must still suppress every matching old version from the high levels.
    ASSERT_OK_AND_ASSIGN(std::vector<Row> candidates, ReadRows(plan, realtime_context, predicate,
                                                               /*enable_predicate_filter=*/false));
    ASSERT_EQ((std::vector<Row>{{1, "level0-current-one", "p0"},
                                {3, "memory-current-three", "p0"},
                                {4, "base-four", "p0"},
                                {7, matching_payload, "p0"}}),
              candidates);

    std::shared_ptr<Predicate> matching_key_predicate = PredicateBuilder::Equal(
        /*field_index=*/0, /*field_name=*/"id", FieldType::BIGINT, Literal(int64_t{1}));
    ASSERT_OK_AND_ASSIGN(std::shared_ptr<Plan> matching_key_plan,
                         CreatePlan(realtime_context, matching_key_predicate));
    ASSERT_OK_AND_ASSIGN(std::vector<Row> matching_key_candidates,
                         ReadRows(matching_key_plan, realtime_context, matching_key_predicate,
                                  /*enable_predicate_filter=*/true));
    ASSERT_EQ((std::vector<Row>{{1, "level0-current-one", "p0"}}), matching_key_candidates);

    std::shared_ptr<Predicate> deleted_key_predicate = PredicateBuilder::Equal(
        /*field_index=*/0, /*field_name=*/"id", FieldType::BIGINT, Literal(int64_t{2}));
    ASSERT_OK_AND_ASSIGN(std::shared_ptr<Plan> deleted_key_plan,
                         CreatePlan(realtime_context, deleted_key_predicate));
    ASSERT_OK_AND_ASSIGN(std::vector<Row> deleted_key_candidates,
                         ReadRows(deleted_key_plan, realtime_context, deleted_key_predicate,
                                  /*enable_predicate_filter=*/true));
    ASSERT_TRUE(deleted_key_candidates.empty());
    ASSERT_OK(PrepareAndClose(writer.get()));
}

TEST_F(RealtimeWriteInteTest, TestPkDvPushesOnlyPrimaryKeyPredicatesIntoLevel0AndMemory) {
    options_[Options::FILE_FORMAT] = "parquet";
    options_[Options::DELETION_VECTORS_ENABLED] = "true";
    options_[Options::REALTIME_STORE_STATS_MODE] = "full";
    options_[Options::WRITE_BATCH_SIZE] = "1";
    options_["parquet.page.size"] = "1";
    options_["parquet.write.enable-page-index"] = "true";
    options_["parquet.write.max-row-group-length"] = "1";
    options_["parquet.read.enable-page-index-filter"] = "true";
    CreatePkTable();
    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
                         RealtimeContext::Create());
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
                         CreateRealtimeWriter(realtime_context));

    auto write_one = [&](const Row& row, RecordBatch::RowKind row_kind) -> Status {
        PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<RecordBatch> batch,
                               MakeBatch({row}, /*partitioned=*/false, /*bucket=*/0, {row_kind}));
        return writer->Write(std::move(batch));
    };

    ASSERT_OK(write_one({1, "high-match", "p0"}, RecordBatch::RowKind::INSERT));
    ASSERT_OK(write_one({2, "high-other", "p0"}, RecordBatch::RowKind::INSERT));
    ASSERT_OK(write_one({3, "high-old", "p0"}, RecordBatch::RowKind::INSERT));
    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> base_progress,
                         writer->PrepareCommitWithProgress(/*commit_identifier=*/0));
    ASSERT_OK_AND_ASSIGN(int64_t base_snapshot_id, Commit(base_progress, /*commit_identifier=*/0));
    ASSERT_OK(writer->RefreshCommittedSnapshot(base_snapshot_id));
    ASSERT_OK_AND_ASSIGN(Snapshot full_compact_snapshot,
                         CompactAndCommit(/*partition=*/{}, /*bucket=*/0,
                                          /*commit_identifier=*/1));
    ASSERT_OK(writer->RefreshCommittedSnapshot(full_compact_snapshot.Id()));

    ASSERT_OK(write_one({3, "high-current", "p0"}, RecordBatch::RowKind::UPDATE_AFTER));
    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> update_progress,
                         writer->PrepareCommitWithProgress(/*commit_identifier=*/2));
    ASSERT_OK_AND_ASSIGN(int64_t update_snapshot_id,
                         Commit(update_progress, /*commit_identifier=*/2));
    ASSERT_OK(writer->RefreshCommittedSnapshot(update_snapshot_id));
    ASSERT_OK_AND_ASSIGN(Snapshot dv_compact_snapshot,
                         CompactAndCommit(/*partition=*/{}, /*bucket=*/0,
                                          /*commit_identifier=*/3,
                                          /*full_compaction=*/false));
    ASSERT_OK(writer->RefreshCommittedSnapshot(dv_compact_snapshot.Id()));

    ASSERT_OK(write_one({4, "level0-match", "p0"}, RecordBatch::RowKind::INSERT));
    ASSERT_OK(write_one({5, "level0-other", "p0"}, RecordBatch::RowKind::INSERT));
    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> level0_progress,
                         writer->PrepareCommitWithProgress(/*commit_identifier=*/4));
    ASSERT_OK_AND_ASSIGN(int64_t level0_snapshot_id,
                         Commit(level0_progress, /*commit_identifier=*/4));
    ASSERT_OK(writer->RefreshCommittedSnapshot(level0_snapshot_id));

    ASSERT_OK(write_one({6, "memory-match", "p0"}, RecordBatch::RowKind::INSERT));
    ASSERT_OK(write_one({7, "memory-other", "p0"}, RecordBatch::RowKind::INSERT));

    ASSERT_OK_AND_ASSIGN(std::shared_ptr<Plan> unfiltered_plan,
                         CreatePlan(realtime_context, /*predicate=*/nullptr));
    ASSERT_EQ(1, unfiltered_plan->Splits().size());
    std::shared_ptr<RealtimeSplit> realtime_split =
        std::dynamic_pointer_cast<RealtimeSplit>(unfiltered_plan->Splits()[0]);
    ASSERT_NE(nullptr, realtime_split);
    ASSERT_GT(realtime_split->MemoryEndOffset(), realtime_split->CommittedEndOffset());
    ASSERT_OK_AND_ASSIGN(DiskFileLayout layout, InspectDiskFiles(realtime_split->DiskSplits()));
    ASSERT_TRUE(layout.has_level0_file);
    ASSERT_TRUE(layout.has_high_level_file);
    ASSERT_TRUE(layout.has_high_level_deletion_vector);

    // Keep residual filtering disabled so candidate rows expose the actual source pushdown:
    // high-level files receive the full predicate, while L0 and memory receive only PK predicates.
    auto read_candidates =
        [&](const std::shared_ptr<Predicate>& predicate) -> Result<std::vector<Row>> {
        PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<Plan> plan, CreatePlan(realtime_context, predicate));
        return ReadRows(plan, realtime_context, predicate,
                        /*enable_predicate_filter=*/false);
    };

    std::shared_ptr<Predicate> level0_key_predicate = PredicateBuilder::Equal(
        /*field_index=*/0, /*field_name=*/"id", FieldType::BIGINT, Literal(int64_t{4}));
    ASSERT_OK_AND_ASSIGN(std::vector<Row> level0_key_candidates,
                         read_candidates(level0_key_predicate));
    ASSERT_EQ((std::vector<Row>{{4, "level0-match", "p0"}}), level0_key_candidates);

    std::shared_ptr<Predicate> memory_key_predicate = PredicateBuilder::Equal(
        /*field_index=*/0, /*field_name=*/"id", FieldType::BIGINT, Literal(int64_t{6}));
    ASSERT_OK_AND_ASSIGN(std::vector<Row> memory_key_candidates,
                         read_candidates(memory_key_predicate));
    ASSERT_EQ((std::vector<Row>{{6, "memory-match", "p0"}}), memory_key_candidates);

    const std::string high_match = "high-match";
    std::shared_ptr<Predicate> non_key_predicate = PredicateBuilder::Equal(
        /*field_index=*/1, /*field_name=*/"payload", FieldType::STRING,
        Literal(FieldType::STRING, high_match.data(), high_match.size()));
    ASSERT_OK_AND_ASSIGN(std::vector<Row> non_key_candidates, read_candidates(non_key_predicate));
    ASSERT_EQ((std::vector<Row>{{1, "high-match", "p0"},
                                {4, "level0-match", "p0"},
                                {5, "level0-other", "p0"},
                                {6, "memory-match", "p0"},
                                {7, "memory-other", "p0"}}),
              non_key_candidates);

    ASSERT_OK_AND_ASSIGN(
        std::shared_ptr<Predicate> mixed_predicate,
        PredicateBuilder::And(
            {PredicateBuilder::GreaterOrEqual(/*field_index=*/0, /*field_name=*/"id",
                                              FieldType::BIGINT, Literal(int64_t{4})),
             PredicateBuilder::Equal(
                 /*field_index=*/1, /*field_name=*/"payload", FieldType::STRING,
                 Literal(FieldType::STRING, high_match.data(), high_match.size()))}));
    ASSERT_OK_AND_ASSIGN(std::vector<Row> mixed_candidates, read_candidates(mixed_predicate));
    ASSERT_EQ((std::vector<Row>{{4, "level0-match", "p0"},
                                {5, "level0-other", "p0"},
                                {6, "memory-match", "p0"},
                                {7, "memory-other", "p0"}}),
              mixed_candidates);
    ASSERT_OK(PrepareAndClose(writer.get()));
}

TEST_F(RealtimeWriteInteTest, TestPkDvMergesDuplicateKeysAcrossLevel0RunsAndMemory) {
    options_[Options::FILE_FORMAT] = "parquet";
    options_[Options::DELETION_VECTORS_ENABLED] = "true";
    CreatePkTable();
    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
                         RealtimeContext::Create());
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
                         CreateRealtimeWriter(realtime_context));

    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> first_level0_batch,
                         MakeBatch({Row{1, "level0-old-one", "p0"}, Row{2, "level0-old-two", "p0"},
                                    Row{3, "level0-stable-three", "p0"}},
                                   /*partitioned=*/false));
    ASSERT_OK(writer->Write(std::move(first_level0_batch)));
    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> first_level0_progress,
                         writer->PrepareCommitWithProgress(/*commit_identifier=*/0));
    ASSERT_OK_AND_ASSIGN(int64_t first_level0_snapshot_id,
                         Commit(first_level0_progress, /*commit_identifier=*/0));
    ASSERT_OK(writer->RefreshCommittedSnapshot(first_level0_snapshot_id));

    ASSERT_OK_AND_ASSIGN(
        std::unique_ptr<RecordBatch> second_level0_batch,
        MakeBatch({Row{1, "level0-new-one", "p0"}, Row{2, "level0-deleted-two", "p0"}},
                  /*partitioned=*/false, /*bucket=*/0,
                  {RecordBatch::RowKind::UPDATE_AFTER, RecordBatch::RowKind::DELETE}));
    ASSERT_OK(writer->Write(std::move(second_level0_batch)));
    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> second_level0_progress,
                         writer->PrepareCommitWithProgress(/*commit_identifier=*/1));
    ASSERT_OK_AND_ASSIGN(int64_t second_level0_snapshot_id,
                         Commit(second_level0_progress, /*commit_identifier=*/1));
    ASSERT_OK(writer->RefreshCommittedSnapshot(second_level0_snapshot_id));

    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> memory_batch,
                         MakeBatch({Row{1, "memory-new-one", "p0"}}, /*partitioned=*/false,
                                   /*bucket=*/0, {RecordBatch::RowKind::UPDATE_AFTER}));
    ASSERT_OK(writer->Write(std::move(memory_batch)));

    ASSERT_OK_AND_ASSIGN(std::shared_ptr<Plan> plan,
                         CreatePlan(realtime_context, /*predicate=*/nullptr));
    ASSERT_EQ(1, plan->Splits().size());
    std::shared_ptr<RealtimeSplit> realtime_split =
        std::dynamic_pointer_cast<RealtimeSplit>(plan->Splits()[0]);
    ASSERT_NE(nullptr, realtime_split);
    size_t level0_file_count = 0;
    for (const std::shared_ptr<Split>& disk_split : realtime_split->DiskSplits()) {
        std::shared_ptr<DataSplitImpl> data_split =
            std::dynamic_pointer_cast<DataSplitImpl>(disk_split);
        ASSERT_NE(nullptr, data_split);
        for (const std::shared_ptr<DataFileMeta>& file : data_split->DataFiles()) {
            if (file->level == 0) {
                ++level0_file_count;
            }
        }
    }
    ASSERT_GE(level0_file_count, 2);

    ASSERT_OK_AND_ASSIGN(std::vector<Row> actual_rows, ReadRows(plan, realtime_context));
    ASSERT_EQ((std::vector<Row>{{1, "memory-new-one", "p0"}, {3, "level0-stable-three", "p0"}}),
              actual_rows);
    ASSERT_OK(PrepareAndClose(writer.get()));
}

TEST_F(RealtimeWriteInteTest, TestRealtimeOffsetFileLifecycle) {
    options_[Options::SNAPSHOT_NUM_RETAINED_MIN] = "1";
    options_[Options::SNAPSHOT_NUM_RETAINED_MAX] = "1";
    options_[Options::SNAPSHOT_TIME_RETAINED] = "1ms";
    CreateTable(/*partition_keys=*/{});
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer, CreateRealtimeWriter());

    std::vector<Row> first_rows = MakeRows(/*first_id=*/0, /*count=*/3, /*partition=*/"p0");
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> first_batch,
                         MakeBatch(first_rows, /*partitioned=*/false));
    ASSERT_OK(writer->Write(std::move(first_batch)));
    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> first_commits,
                         writer->PrepareCommitWithProgress(/*commit_identifier=*/0));
    ASSERT_OK_AND_ASSIGN(int64_t first_snapshot_id, Commit(first_commits, /*commit_identifier=*/0));

    std::vector<Row> second_rows = MakeRows(/*first_id=*/3, /*count=*/2, /*partition=*/"p0");
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> second_batch,
                         MakeBatch(second_rows, /*partitioned=*/false));
    ASSERT_OK(writer->Write(std::move(second_batch)));
    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> second_commits,
                         writer->PrepareCommitWithProgress(/*commit_identifier=*/1));
    ASSERT_OK_AND_ASSIGN(int64_t second_snapshot_id,
                         Commit(second_commits, /*commit_identifier=*/1));

    ASSERT_OK_AND_ASSIGN(CoreOptions core_options, CoreOptions::FromMap(options_));
    std::shared_ptr<FileSystem> file_system = core_options.GetFileSystem();
    SnapshotManager snapshot_manager(file_system, table_path_);
    ASSERT_OK_AND_ASSIGN(Snapshot first_snapshot, snapshot_manager.LoadSnapshot(first_snapshot_id));
    ASSERT_OK_AND_ASSIGN(Snapshot second_snapshot,
                         snapshot_manager.LoadSnapshot(second_snapshot_id));
    std::optional<std::string> first_offsets_path =
        RealtimeCommitProperties::GetOffsetsPath(first_snapshot);
    std::optional<std::string> second_offsets_path =
        RealtimeCommitProperties::GetOffsetsPath(second_snapshot);
    ASSERT_TRUE(first_offsets_path);
    ASSERT_TRUE(second_offsets_path);
    ASSERT_NE(first_offsets_path, second_offsets_path);

    std::string orphan_offsets_path = PathUtil::JoinPath(
        RealtimeCommitProperties::OffsetsDirectory(table_path_, core_options.GetBranch()),
        "orphan.offsets");
    ASSERT_OK(file_system->WriteFile(orphan_offsets_path, "orphan", /*overwrite=*/false));
    CleanContextBuilder clean_builder(table_path_);
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<CleanContext> clean_context,
                         clean_builder.WithFileSystem(file_system)
                             .WithOlderThanMs(std::numeric_limits<int64_t>::max())
                             .Finish());
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<OrphanFilesCleaner> cleaner,
                         OrphanFilesCleaner::Create(std::move(clean_context)));
    ASSERT_OK_AND_ASSIGN(std::set<std::string> cleaned_paths, cleaner->Clean());
    ASSERT_EQ(std::set<std::string>({orphan_offsets_path}), cleaned_paths);
    ASSERT_OK_AND_ASSIGN(bool first_offsets_exist, file_system->Exists(first_offsets_path.value()));
    ASSERT_TRUE(first_offsets_exist);
    ASSERT_OK_AND_ASSIGN(bool second_offsets_exist,
                         file_system->Exists(second_offsets_path.value()));
    ASSERT_TRUE(second_offsets_exist);

    ASSERT_OK_AND_ASSIGN(int32_t expired_snapshots, ExpireSnapshots());
    ASSERT_EQ(1, expired_snapshots);
    ASSERT_OK_AND_ASSIGN(first_offsets_exist, file_system->Exists(first_offsets_path.value()));
    ASSERT_FALSE(first_offsets_exist);
    ASSERT_OK_AND_ASSIGN(second_offsets_exist, file_system->Exists(second_offsets_path.value()));
    ASSERT_TRUE(second_offsets_exist);
    ASSERT_OK_AND_ASSIGN(RealtimeOffsetMap committed_offsets,
                         RealtimeCommitProperties::ReadOffsets(second_snapshot, file_system));
    ASSERT_EQ(5, committed_offsets.at(RealtimePartitionBucket(/*partition=*/{}, /*bucket=*/0)));
    ASSERT_OK(writer->Close());
}

TEST_F(RealtimeWriteInteTest, TestCompactionSnapshotRetainsSharedOffsetFile) {
    options_[Options::TARGET_FILE_ROW_NUM] = "2";
    options_[Options::COMPACTION_MIN_FILE_NUM] = "2";
    options_[Options::SNAPSHOT_NUM_RETAINED_MIN] = "1";
    options_[Options::SNAPSHOT_NUM_RETAINED_MAX] = "1";
    options_[Options::SNAPSHOT_TIME_RETAINED] = "1ms";
    CreateTable(/*partition_keys=*/{});
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer, CreateRealtimeWriter());

    for (int64_t first_id = 0; first_id < 5; first_id += 2) {
        std::vector<Row> rows =
            MakeRows(first_id, std::min<int64_t>(2, 5 - first_id), /*partition=*/"p0");
        ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> batch,
                             MakeBatch(rows, /*partitioned=*/false));
        ASSERT_OK(writer->Write(std::move(batch)));
    }
    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> commits,
                         writer->PrepareCommitWithProgress(/*commit_identifier=*/0));
    ASSERT_OK_AND_ASSIGN(int64_t realtime_snapshot_id, Commit(commits, /*commit_identifier=*/0));

    ASSERT_OK_AND_ASSIGN(Snapshot compact_snapshot, CompactAndCommit(/*partition=*/{}, /*bucket=*/0,
                                                                     /*commit_identifier=*/1));
    ASSERT_EQ(Snapshot::CommitKind::Compact(), compact_snapshot.GetCommitKind());

    ASSERT_OK_AND_ASSIGN(CoreOptions core_options, CoreOptions::FromMap(options_));
    std::shared_ptr<FileSystem> file_system = core_options.GetFileSystem();
    SnapshotManager snapshot_manager(file_system, table_path_);
    ASSERT_OK_AND_ASSIGN(Snapshot realtime_snapshot,
                         snapshot_manager.LoadSnapshot(realtime_snapshot_id));
    std::optional<std::string> realtime_offsets_path =
        RealtimeCommitProperties::GetOffsetsPath(realtime_snapshot);
    std::optional<std::string> compact_offsets_path =
        RealtimeCommitProperties::GetOffsetsPath(compact_snapshot);
    ASSERT_TRUE(realtime_offsets_path);
    ASSERT_TRUE(compact_offsets_path);
    ASSERT_EQ(realtime_offsets_path, compact_offsets_path);

    ASSERT_OK_AND_ASSIGN(int32_t expired_snapshots, ExpireSnapshots());
    ASSERT_EQ(1, expired_snapshots);
    ASSERT_OK_AND_ASSIGN(bool offsets_exist, file_system->Exists(compact_offsets_path.value()));
    ASSERT_TRUE(offsets_exist);
    ASSERT_OK_AND_ASSIGN(RealtimeOffsetMap committed_offsets,
                         RealtimeCommitProperties::ReadOffsets(compact_snapshot, file_system));
    ASSERT_EQ(5, committed_offsets.at(RealtimePartitionBucket(/*partition=*/{}, /*bucket=*/0)));
    ASSERT_OK(writer->Close());
}

TEST_F(RealtimeWriteInteTest, TestReadMemoryBeforePrepareCommit) {
    CreateTable(/*partition_keys=*/{});
    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
                         RealtimeContext::Create());
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
                         CreateRealtimeWriter(realtime_context));
    std::vector<Row> rows = MakeRows(/*first_id=*/0, /*count=*/10, /*partition=*/"p0");
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> batch,
                         MakeBatch(rows, /*partitioned=*/false));
    ASSERT_OK(writer->Write(std::move(batch)));

    ASSERT_OK_AND_ASSIGN(std::shared_ptr<Plan> plan,
                         CreatePlan(realtime_context, /*predicate=*/nullptr));
    ASSERT_EQ(1, plan->Splits().size());
    std::shared_ptr<RealtimeSplit> realtime_split =
        std::dynamic_pointer_cast<RealtimeSplit>(plan->Splits()[0]);
    ASSERT_NE(nullptr, realtime_split);
    ASSERT_EQ(RealtimeSplit::kCurrentVersion, realtime_split->Version());
    ASSERT_FALSE(realtime_split->SnapshotId().has_value());
    ASSERT_EQ(0, realtime_split->CommittedEndOffset());
    ASSERT_EQ(10, realtime_split->MemoryEndOffset());
    ASSERT_FALSE(realtime_split->OpaqueTicket().empty());
    ASSERT_NOK_WITH_MSG(ReadRows(plan, /*realtime_context=*/nullptr),
                        "requires a real-time context");
    ASSERT_OK_AND_ASSIGN(std::vector<Row> actual_rows, ReadRows(plan, realtime_context));
    ASSERT_EQ(rows, actual_rows);
    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContextImpl> realtime_context_impl,
                         RealtimeContextImpl::Cast(realtime_context));
    ASSERT_NOK_WITH_MSG(realtime_context_impl->ResolveReadView(realtime_split->OpaqueTicket()),
                        "ticket does not exist or has expired");
    ASSERT_OK(PrepareAndClose(writer.get()));
}

TEST_F(RealtimeWriteInteTest, TestPlanExcludesRowsWrittenAfterMemoryEndOffset) {
    CreateTable(/*partition_keys=*/{});
    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
                         RealtimeContext::Create());
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
                         CreateRealtimeWriter(realtime_context));

    std::vector<Row> first_rows = MakeRows(/*first_id=*/0, /*count=*/10, /*partition=*/"p0");
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> first_batch,
                         MakeBatch(first_rows, /*partitioned=*/false));
    ASSERT_OK(writer->Write(std::move(first_batch)));
    ASSERT_OK_AND_ASSIGN(std::shared_ptr<Plan> first_plan,
                         CreatePlan(realtime_context, /*predicate=*/nullptr));
    ASSERT_EQ(1, first_plan->Splits().size());
    std::shared_ptr<RealtimeSplit> first_realtime_split =
        std::dynamic_pointer_cast<RealtimeSplit>(first_plan->Splits()[0]);
    ASSERT_NE(nullptr, first_realtime_split);
    ASSERT_EQ(10, first_realtime_split->MemoryEndOffset());

    std::vector<Row> second_rows = MakeRows(/*first_id=*/10, /*count=*/5, /*partition=*/"p0");
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> second_batch,
                         MakeBatch(second_rows, /*partitioned=*/false));
    ASSERT_OK(writer->Write(std::move(second_batch)));

    ASSERT_OK_AND_ASSIGN(std::vector<Row> first_actual_rows,
                         ReadRows(first_plan, realtime_context));
    ASSERT_EQ(first_rows, first_actual_rows);

    ASSERT_OK_AND_ASSIGN(std::shared_ptr<Plan> second_plan,
                         CreatePlan(realtime_context, /*predicate=*/nullptr));
    ASSERT_EQ(1, second_plan->Splits().size());
    std::shared_ptr<RealtimeSplit> second_realtime_split =
        std::dynamic_pointer_cast<RealtimeSplit>(second_plan->Splits()[0]);
    ASSERT_NE(nullptr, second_realtime_split);
    ASSERT_EQ(15, second_realtime_split->MemoryEndOffset());

    std::vector<Row> expected_rows = first_rows;
    expected_rows.insert(expected_rows.end(), second_rows.begin(), second_rows.end());
    ASSERT_OK_AND_ASSIGN(std::vector<Row> second_actual_rows,
                         ReadRows(second_plan, realtime_context));
    ASSERT_EQ(expected_rows, second_actual_rows);
    ASSERT_OK(PrepareAndClose(writer.get()));
}

TEST_F(RealtimeWriteInteTest, TestReadFailsAfterRealtimeSplitTicketExpires) {
    options_[Options::REALTIME_READ_VIEW_TTL] = "10 ms";
    CreateTable(/*partition_keys=*/{});
    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
                         RealtimeContext::Create());
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
                         CreateRealtimeWriter(realtime_context));
    std::vector<Row> rows = MakeRows(/*first_id=*/0, /*count=*/3, /*partition=*/"p0");
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> batch,
                         MakeBatch(rows, /*partitioned=*/false));
    ASSERT_OK(writer->Write(std::move(batch)));
    ASSERT_OK_AND_ASSIGN(std::shared_ptr<Plan> plan,
                         CreatePlan(realtime_context, /*predicate=*/nullptr));

    std::this_thread::sleep_for(std::chrono::milliseconds(50));
    ASSERT_NOK_WITH_MSG(ReadRows(plan, realtime_context), "ticket does not exist or has expired");
    ASSERT_OK(PrepareAndClose(writer.get()));
}

TEST_F(RealtimeWriteInteTest, TestSuccessfulReaderCreationConsumesRealtimeSplitTicket) {
    CreateTable(/*partition_keys=*/{});
    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
                         RealtimeContext::Create());
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
                         CreateRealtimeWriter(realtime_context));
    std::vector<Row> rows = MakeRows(/*first_id=*/0, /*count=*/3, /*partition=*/"p0");
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> batch,
                         MakeBatch(rows, /*partitioned=*/false));
    ASSERT_OK(writer->Write(std::move(batch)));
    ASSERT_OK_AND_ASSIGN(std::shared_ptr<Plan> plan,
                         CreatePlan(realtime_context, /*predicate=*/nullptr));

    ReadContextBuilder read_builder(table_path_);
    read_builder.SetOptions(options_)
        .SetReadFieldNames({"id", "payload", "pt"})
        .WithRealtimeContext(realtime_context)
        .WithMemoryPool(pool_);
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<ReadContext> read_context, read_builder.Finish());
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<TableRead> table_read,
                         TableRead::Create(std::move(read_context)));
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<BatchReader> first_reader,
                         table_read->CreateReader(plan->Splits()));
    first_reader->Close();
    first_reader.reset();

    ASSERT_NOK_WITH_MSG(table_read->CreateReader(plan->Splits()),
                        "ticket does not exist or has expired");
    ASSERT_OK(PrepareAndClose(writer.get()));
}

TEST_F(RealtimeWriteInteTest, TestFailedReaderCreationPreservesRealtimeSplitTicket) {
    CreateTable(/*partition_keys=*/{});
    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
                         RealtimeContext::Create());
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
                         CreateRealtimeWriter(realtime_context));
    std::vector<Row> rows = MakeRows(/*first_id=*/0, /*count=*/3, /*partition=*/"p0");
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> batch,
                         MakeBatch(rows, /*partitioned=*/false));
    ASSERT_OK(writer->Write(std::move(batch)));
    ASSERT_OK_AND_ASSIGN(std::shared_ptr<Plan> plan,
                         CreatePlan(realtime_context, /*predicate=*/nullptr));
    ASSERT_EQ(1, plan->Splits().size());

    ReadContextBuilder read_builder(table_path_);
    read_builder.SetOptions(options_)
        .SetReadFieldNames({"id", "payload", "pt"})
        .SetPredicate(std::make_shared<UnsupportedPredicate>())
        .EnablePredicateFilter(true)
        .WithRealtimeContext(realtime_context)
        .WithMemoryPool(pool_);
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<ReadContext> read_context, read_builder.Finish());
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<TableRead> table_read,
                         TableRead::Create(std::move(read_context)));
    ASSERT_NOK_WITH_MSG(table_read->CreateReader(plan->Splits()),
                        "cannot cast predicate unsupported");

    ASSERT_OK_AND_ASSIGN(std::vector<Row> actual_rows, ReadRows(plan, realtime_context));
    ASSERT_EQ(rows, actual_rows);
    ASSERT_OK(PrepareAndClose(writer.get()));
}

TEST_F(RealtimeWriteInteTest, TestCloseWriterWithBuildingDataInvalidatesContext) {
    CreateTable(/*partition_keys=*/{});
    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
                         RealtimeContext::Create());
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
                         CreateRealtimeWriter(realtime_context));
    std::vector<Row> rows = MakeRows(/*first_id=*/0, /*count=*/3, /*partition=*/"p0");
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> batch,
                         MakeBatch(rows, /*partitioned=*/false));
    ASSERT_OK(writer->Write(std::move(batch)));
    ASSERT_NOK_WITH_MSG(writer->Close(), "not covered by a successful");

    ASSERT_NOK_WITH_MSG(ReadRows(realtime_context), "real-time context cannot be reused");
    ASSERT_NOK_WITH_MSG(CreateRealtimeWriter(realtime_context),
                        "real-time context cannot be reused");
}

TEST_F(RealtimeWriteInteTest, TestCloseWriterWithSealedDataInvalidatesContext) {
    CreateTable(/*partition_keys=*/{});
    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
                         RealtimeContext::Create());
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
                         CreateRealtimeWriter(realtime_context));
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> batch,
                         MakeBatch(MakeRows(/*first_id=*/0, /*count=*/3, /*partition=*/"p0"),
                                   /*partitioned=*/false));
    ASSERT_OK(writer->Write(std::move(batch)));
    ASSERT_OK(writer->Seal());
    ASSERT_NOK_WITH_MSG(writer->Close(), "not covered by a successful");

    ASSERT_NOK_WITH_MSG(CreateRealtimeWriter(realtime_context),
                        "real-time context cannot be reused");
}

TEST_F(RealtimeWriteInteTest, TestPinnedPlanRemainsReadableAfterWriterClose) {
    CreateTable(/*partition_keys=*/{});
    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
                         RealtimeContext::Create());
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
                         CreateRealtimeWriter(realtime_context));
    std::vector<Row> rows = MakeRows(/*first_id=*/0, /*count=*/3, /*partition=*/"p0");
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> batch,
                         MakeBatch(rows, /*partitioned=*/false));
    ASSERT_OK(writer->Write(std::move(batch)));
    ASSERT_OK_AND_ASSIGN(std::shared_ptr<Plan> plan,
                         CreatePlan(realtime_context, /*predicate=*/nullptr));
    ASSERT_OK(PrepareAndClose(writer.get()));

    ASSERT_OK_AND_ASSIGN(std::vector<Row> actual_rows, ReadRows(plan, realtime_context));
    ASSERT_EQ(rows, actual_rows);
}

TEST_F(RealtimeWriteInteTest, TestCloseWriterAllowsContextReuseByLaterWriter) {
    CreateTable(/*partition_keys=*/{});
    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
                         RealtimeContext::Create());
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> first_writer,
                         CreateRealtimeWriter(realtime_context));
    std::vector<Row> first_rows = MakeRows(/*first_id=*/0, /*count=*/3, /*partition=*/"p0");
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> first_batch,
                         MakeBatch(first_rows, /*partitioned=*/false));
    ASSERT_OK(first_writer->Write(std::move(first_batch)));
    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> commits,
                         first_writer->PrepareCommitWithProgress(/*commit_identifier=*/0));
    ASSERT_EQ(1, commits.size());
    ASSERT_EQ(OffsetRange(0, 3), commits[0].offset_range);
    ASSERT_OK(first_writer->Close());

    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> second_writer,
                         CreateRealtimeWriter(realtime_context));
    std::vector<Row> second_rows = MakeRows(/*first_id=*/3, /*count=*/2, /*partition=*/"p0");
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> second_batch,
                         MakeBatch(second_rows, /*partitioned=*/false));
    ASSERT_OK(second_writer->Write(std::move(second_batch)));
    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> second_commits,
                         second_writer->PrepareCommitWithProgress(/*commit_identifier=*/1));
    ASSERT_EQ(1, second_commits.size());
    ASSERT_EQ(OffsetRange(3, 5), second_commits[0].offset_range);

    commits.push_back(std::move(second_commits[0]));
    ASSERT_OK(Commit(commits, /*commit_identifier=*/1));
    std::vector<Row> expected_rows = first_rows;
    expected_rows.insert(expected_rows.end(), second_rows.begin(), second_rows.end());
    ASSERT_OK_AND_ASSIGN(std::vector<Row> actual_rows, ReadRows(realtime_context));
    ASSERT_EQ(expected_rows, actual_rows);
    ASSERT_OK(second_writer->Close());
}

TEST_F(RealtimeWriteInteTest, TestReadCommittedDiskAndBuildingMemory) {
    CreateTable(/*partition_keys=*/{});
    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
                         RealtimeContext::Create());
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
                         CreateRealtimeWriter(realtime_context));

    std::vector<Row> disk_rows = MakeRows(/*first_id=*/0, /*count=*/3, /*partition=*/"p0");
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> disk_batch,
                         MakeBatch(disk_rows, /*partitioned=*/false));
    ASSERT_OK(writer->Write(std::move(disk_batch)));
    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> disk_commits,
                         writer->PrepareCommitWithProgress(/*commit_identifier=*/0));
    ASSERT_EQ(1, disk_commits.size());
    ASSERT_EQ(OffsetRange(0, 3), disk_commits[0].offset_range);

    std::vector<Row> memory_rows = MakeRows(/*first_id=*/3, /*count=*/2, /*partition=*/"p0");
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> memory_batch,
                         MakeBatch(memory_rows, /*partitioned=*/false));
    ASSERT_OK(writer->Write(std::move(memory_batch)));
    ASSERT_OK(Commit(disk_commits, /*commit_identifier=*/0));

    std::vector<Row> expected_rows = disk_rows;
    expected_rows.insert(expected_rows.end(), memory_rows.begin(), memory_rows.end());
    ASSERT_OK_AND_ASSIGN(std::vector<Row> actual_rows, ReadRows(realtime_context));
    ASSERT_EQ(expected_rows, actual_rows);
    ASSERT_OK(PrepareAndClose(writer.get()));
}

TEST_F(RealtimeWriteInteTest, TestCountReaderDoesNotSupportRealtimeSplit) {
    CreateTable(/*partition_keys=*/{});
    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
                         RealtimeContext::Create());
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
                         CreateRealtimeWriter(realtime_context));

    std::vector<Row> rows = MakeRows(/*first_id=*/0, /*count=*/3, /*partition=*/"p0");
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> batch,
                         MakeBatch(rows, /*partitioned=*/false));
    ASSERT_OK(writer->Write(std::move(batch)));
    ASSERT_OK_AND_ASSIGN(std::shared_ptr<Plan> plan,
                         CreatePlan(realtime_context, /*predicate=*/nullptr));
    ASSERT_NOK_WITH_MSG(CountRows(plan, realtime_context),
                        "does not support process-local real-time splits");
    ASSERT_OK(PrepareAndClose(writer.get()));
}

TEST_F(RealtimeWriteInteTest, TestProjectionAndPredicateForMemoryAndDisk) {
    CreateTable(/*partition_keys=*/{});
    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
                         RealtimeContext::Create());
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
                         CreateRealtimeWriter(realtime_context));
    std::shared_ptr<Predicate> scan_predicate =
        PredicateBuilder::GreaterThan(/*field_index=*/0, /*field_name=*/"id", FieldType::BIGINT,
                                      Literal(static_cast<int64_t>(1)));
    std::shared_ptr<Predicate> read_predicate =
        PredicateBuilder::GreaterThan(/*field_index=*/1, /*field_name=*/"id", FieldType::BIGINT,
                                      Literal(static_cast<int64_t>(1)));
    const std::vector<std::string> read_fields = {"payload", "id"};
    std::shared_ptr<arrow::DataType> result_type = arrow::struct_(
        {arrow::field("_VALUE_KIND", arrow::int8()), arrow::field("payload", arrow::utf8()),
         arrow::field("id", arrow::int64())});

    std::vector<Row> disk_rows = MakeRows(/*first_id=*/0, /*count=*/3, /*partition=*/"p0");
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> disk_batch,
                         MakeBatch(disk_rows, /*partitioned=*/false));
    ASSERT_OK(writer->Write(std::move(disk_batch)));

    ASSERT_OK_AND_ASSIGN(std::shared_ptr<Plan> memory_plan,
                         CreatePlan(realtime_context, scan_predicate));
    ASSERT_OK_AND_ASSIGN(std::shared_ptr<arrow::ChunkedArray> memory_result,
                         ReadPlan(memory_plan, realtime_context, read_fields, read_predicate,
                                  /*enable_predicate_filter=*/true));
    std::shared_ptr<arrow::Array> expected_memory =
        arrow::ipc::internal::json::ArrayFromJSON(result_type, R"([
            [0, "value-2", 2]
        ])")
            .ValueOrDie();
    ASSERT_NE(nullptr, memory_result);
    ASSERT_TRUE(std::make_shared<arrow::ChunkedArray>(expected_memory)->Equals(*memory_result));

    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> disk_commits,
                         writer->PrepareCommitWithProgress(/*commit_identifier=*/0));
    ASSERT_OK(Commit(disk_commits, /*commit_identifier=*/0));
    std::vector<Row> memory_rows = MakeRows(/*first_id=*/3, /*count=*/3, /*partition=*/"p0");
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> memory_batch,
                         MakeBatch(memory_rows, /*partitioned=*/false));
    ASSERT_OK(writer->Write(std::move(memory_batch)));

    ASSERT_OK_AND_ASSIGN(std::shared_ptr<Plan> union_plan,
                         CreatePlan(realtime_context, scan_predicate));
    ASSERT_OK_AND_ASSIGN(std::shared_ptr<arrow::ChunkedArray> union_result,
                         ReadPlan(union_plan, realtime_context, read_fields, read_predicate,
                                  /*enable_predicate_filter=*/true));
    std::shared_ptr<arrow::Array> expected_union =
        arrow::ipc::internal::json::ArrayFromJSON(result_type, R"([
            [0, "value-2", 2],
            [0, "value-3", 3],
            [0, "value-4", 4],
            [0, "value-5", 5]
        ])")
            .ValueOrDie();
    ASSERT_NE(nullptr, union_result);
    ASSERT_TRUE(std::make_shared<arrow::ChunkedArray>(expected_union)->Equals(*union_result));
    ASSERT_OK(PrepareAndClose(writer.get()));
}

TEST_F(RealtimeWriteInteTest, TestDiskPredicatePushdownWithoutMemoryFiltering) {
    options_[Options::FILE_FORMAT] = "parquet";
    options_[Options::WRITE_BATCH_SIZE] = "1";
    options_["parquet.page.size"] = "1";
    options_["parquet.enable.dictionary"] = "false";
    options_["parquet.write.enable-page-index"] = "true";
    options_["parquet.read.enable-page-index-filter"] = "true";
    CreateTable(/*partition_keys=*/{});
    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
                         RealtimeContext::Create());
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
                         CreateRealtimeWriter(realtime_context));
    std::shared_ptr<Predicate> predicate =
        PredicateBuilder::Equal(/*field_index=*/0, /*field_name=*/"id", FieldType::BIGINT,
                                Literal(static_cast<int64_t>(1)));

    std::vector<Row> disk_rows = MakeRows(/*first_id=*/0, /*count=*/3, /*partition=*/"p0");
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> disk_batch,
                         MakeBatch(disk_rows, /*partitioned=*/false));
    ASSERT_OK(writer->Write(std::move(disk_batch)));
    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> disk_commits,
                         writer->PrepareCommitWithProgress(/*commit_identifier=*/0));
    ASSERT_OK(Commit(disk_commits, /*commit_identifier=*/0));

    std::vector<Row> memory_rows = MakeRows(/*first_id=*/3, /*count=*/3, /*partition=*/"p0");
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> memory_batch,
                         MakeBatch(memory_rows, /*partitioned=*/false));
    ASSERT_OK(writer->Write(std::move(memory_batch)));

    ASSERT_OK_AND_ASSIGN(std::shared_ptr<Plan> plan, CreatePlan(realtime_context, predicate));
    ASSERT_OK_AND_ASSIGN(std::shared_ptr<arrow::ChunkedArray> result,
                         ReadPlan(plan, realtime_context, {"id", "payload", "pt"}, predicate,
                                  /*enable_predicate_filter=*/false));
    std::shared_ptr<arrow::DataType> result_type = arrow::struct_(
        {arrow::field("_VALUE_KIND", arrow::int8()), arrow::field("id", arrow::int64()),
         arrow::field("payload", arrow::utf8()), arrow::field("pt", arrow::utf8())});
    std::shared_ptr<arrow::Array> expected =
        arrow::ipc::internal::json::ArrayFromJSON(result_type, R"([
            [0, 1, "value-1", "p0"],
            [0, 3, "value-3", "p0"],
            [0, 4, "value-4", "p0"],
            [0, 5, "value-5", "p0"]
        ])")
            .ValueOrDie();
    ASSERT_NE(nullptr, result);
    ASSERT_TRUE(std::make_shared<arrow::ChunkedArray>(expected)->Equals(*result))
        << result->ToString();
    ASSERT_OK(PrepareAndClose(writer.get()));
}

TEST_F(RealtimeWriteInteTest, TestMemoryBatchStatisticsPredicatePushdown) {
    CreateTable(/*partition_keys=*/{});
    std::shared_ptr<arrow::DataType> result_type = arrow::struct_(
        {arrow::field("_VALUE_KIND", arrow::int8()), arrow::field("id", arrow::int64()),
         arrow::field("payload", arrow::utf8()), arrow::field("pt", arrow::utf8())});
    auto make_expected = [&](const std::string& json) {
        std::shared_ptr<arrow::Array> array =
            arrow::ipc::internal::json::ArrayFromJSON(result_type, json).ValueOrDie();
        return std::make_shared<arrow::ChunkedArray>(array);
    };
    std::vector<std::shared_ptr<Predicate>> predicates = {
        PredicateBuilder::GreaterThan(/*field_index=*/0, /*field_name=*/"id", FieldType::BIGINT,
                                      Literal(static_cast<int64_t>(100))),
        PredicateBuilder::GreaterThan(/*field_index=*/0, /*field_name=*/"id", FieldType::BIGINT,
                                      Literal(static_cast<int64_t>(5))),
        PredicateBuilder::Equal(/*field_index=*/0, /*field_name=*/"id", FieldType::BIGINT,
                                Literal(static_cast<int64_t>(10))),
    };

    auto check_candidates =
        [&](const std::string& statistics_mode,
            const std::vector<std::shared_ptr<arrow::ChunkedArray>>& expected) -> Status {
        if (expected.size() != predicates.size()) {
            return Status::Invalid("unexpected real-time candidate result count");
        }
        options_[Options::REALTIME_STORE_STATS_MODE] = statistics_mode;
        PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<RealtimeContext> realtime_context,
                               RealtimeContext::Create());
        PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<FileStoreWrite> writer,
                               CreateRealtimeWriter(realtime_context));
        for (int64_t first_id : {0, 10, 20}) {
            std::vector<Row> rows = MakeRows(first_id, /*count=*/3, /*partition=*/"p0");
            PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<RecordBatch> batch,
                                   MakeBatch(rows, /*partitioned=*/false));
            PAIMON_RETURN_NOT_OK(writer->Write(std::move(batch)));
        }

        for (size_t i = 0; i < predicates.size(); ++i) {
            const std::shared_ptr<Predicate>& predicate = predicates[i];
            PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<Plan> plan,
                                   CreatePlan(realtime_context, predicate));
            PAIMON_ASSIGN_OR_RAISE(
                std::shared_ptr<arrow::ChunkedArray> result,
                ReadPlan(plan, realtime_context, {"id", "payload", "pt"}, predicate,
                         /*enable_predicate_filter=*/false));
            std::shared_ptr<arrow::ChunkedArray> actual = result ? result : make_expected("[]");
            if (!expected[i]->Equals(*actual)) {
                return Status::Invalid("unexpected real-time candidate rows: " +
                                       actual->ToString());
            }
        }
        PAIMON_RETURN_NOT_OK(PrepareAndClose(writer.get()));
        return Status::OK();
    };

    std::shared_ptr<arrow::ChunkedArray> all_rows = make_expected(R"([
        [0, 0, "value-0", "p0"],
        [0, 1, "value-1", "p0"],
        [0, 2, "value-2", "p0"],
        [0, 10, "value-10", "p0"],
        [0, 11, "value-11", "p0"],
        [0, 12, "value-12", "p0"],
        [0, 20, "value-20", "p0"],
        [0, 21, "value-21", "p0"],
        [0, 22, "value-22", "p0"]
    ])");
    ASSERT_OK(check_candidates("none", {all_rows, all_rows, all_rows}));

    std::shared_ptr<arrow::ChunkedArray> partially_filtered = make_expected(R"([
        [0, 10, "value-10", "p0"],
        [0, 11, "value-11", "p0"],
        [0, 12, "value-12", "p0"],
        [0, 20, "value-20", "p0"],
        [0, 21, "value-21", "p0"],
        [0, 22, "value-22", "p0"]
    ])");
    std::shared_ptr<arrow::ChunkedArray> matching_batch = make_expected(R"([
        [0, 10, "value-10", "p0"],
        [0, 11, "value-11", "p0"],
        [0, 12, "value-12", "p0"]
    ])");
    ASSERT_OK(check_candidates("full", {make_expected("[]"), partially_filtered, matching_batch}));
}

TEST_F(RealtimeWriteInteTest, TestMemoryBatchStatisticsPredicatePushdownWithDisk) {
    options_[Options::FILE_FORMAT] = "parquet";
    options_[Options::WRITE_BATCH_SIZE] = "1";
    options_[Options::REALTIME_STORE_STATS_MODE] = "full";
    options_["parquet.page.size"] = "1";
    options_["parquet.enable.dictionary"] = "false";
    options_["parquet.write.enable-page-index"] = "true";
    options_["parquet.read.enable-page-index-filter"] = "true";
    CreateTable(/*partition_keys=*/{});
    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
                         RealtimeContext::Create());
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
                         CreateRealtimeWriter(realtime_context));

    std::vector<Row> disk_rows = MakeRows(/*first_id=*/0, /*count=*/6, /*partition=*/"p0");
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> disk_batch,
                         MakeBatch(disk_rows, /*partitioned=*/false));
    ASSERT_OK(writer->Write(std::move(disk_batch)));
    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> disk_commits,
                         writer->PrepareCommitWithProgress(/*commit_identifier=*/0));
    ASSERT_OK(Commit(disk_commits, /*commit_identifier=*/0));

    for (int64_t first_id : {0, 10}) {
        std::vector<Row> rows = MakeRows(first_id, /*count=*/3, /*partition=*/"p0");
        ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> batch,
                             MakeBatch(rows, /*partitioned=*/false));
        ASSERT_OK(writer->Write(std::move(batch)));
    }
    std::shared_ptr<Predicate> predicate =
        PredicateBuilder::GreaterThan(/*field_index=*/0, /*field_name=*/"id", FieldType::BIGINT,
                                      Literal(static_cast<int64_t>(3)));
    ASSERT_OK_AND_ASSIGN(std::shared_ptr<Plan> plan, CreatePlan(realtime_context, predicate));
    ASSERT_OK_AND_ASSIGN(std::shared_ptr<arrow::ChunkedArray> result,
                         ReadPlan(plan, realtime_context, {"id", "payload", "pt"}, predicate,
                                  /*enable_predicate_filter=*/false));

    std::shared_ptr<arrow::DataType> result_type = arrow::struct_(
        {arrow::field("_VALUE_KIND", arrow::int8()), arrow::field("id", arrow::int64()),
         arrow::field("payload", arrow::utf8()), arrow::field("pt", arrow::utf8())});
    std::shared_ptr<arrow::Array> expected =
        arrow::ipc::internal::json::ArrayFromJSON(result_type, R"([
            [0, 4, "value-4", "p0"],
            [0, 5, "value-5", "p0"],
            [0, 10, "value-10", "p0"],
            [0, 11, "value-11", "p0"],
            [0, 12, "value-12", "p0"]
        ])")
            .ValueOrDie();
    ASSERT_NE(nullptr, result);
    ASSERT_TRUE(std::make_shared<arrow::ChunkedArray>(expected)->Equals(*result))
        << result->ToString();
    ASSERT_OK(PrepareAndClose(writer.get()));
}

TEST_F(RealtimeWriteInteTest, TestNullPredicateForMemoryAndDisk) {
    options_[Options::FILE_FORMAT] = "parquet";
    options_[Options::WRITE_BATCH_SIZE] = "1";
    options_[Options::REALTIME_STORE_STATS_MODE] = "full";
    options_["parquet.page.size"] = "1";
    options_["parquet.enable.dictionary"] = "false";
    options_["parquet.write.enable-page-index"] = "true";
    options_["parquet.write.max-row-group-length"] = "1";
    options_["parquet.read.enable-page-index-filter"] = "true";
    CreateTable(/*partition_keys=*/{});
    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
                         RealtimeContext::Create());
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
                         CreateRealtimeWriter(realtime_context));

    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> disk_batch,
                         MakeUnpartitionedBatchFromJson(R"([
                             [0, 0, null, "p0"],
                             [1, 1, "disk-value", "p0"]
                         ])"));
    ASSERT_OK(writer->Write(std::move(disk_batch)));
    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> disk_commits,
                         writer->PrepareCommitWithProgress(/*commit_identifier=*/0));
    ASSERT_OK(Commit(disk_commits, /*commit_identifier=*/0));

    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> non_null_memory_batch,
                         MakeUnpartitionedBatchFromJson(R"([
                             [2, 2, "memory-value-2", "p0"],
                             [3, 3, "memory-value-3", "p0"]
                         ])"));
    ASSERT_OK(writer->Write(std::move(non_null_memory_batch)));
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> nullable_memory_batch,
                         MakeUnpartitionedBatchFromJson(R"([
                             [4, 4, null, "p0"],
                             [5, 5, "memory-value-5", "p0"]
                         ])"));
    ASSERT_OK(writer->Write(std::move(nullable_memory_batch)));

    std::shared_ptr<Predicate> predicate = PredicateBuilder::IsNull(
        /*field_index=*/1, /*field_name=*/"payload", FieldType::STRING);
    ASSERT_OK_AND_ASSIGN(std::shared_ptr<Plan> plan, CreatePlan(realtime_context, predicate));
    ASSERT_OK_AND_ASSIGN(std::shared_ptr<arrow::ChunkedArray> result,
                         ReadPlan(plan, realtime_context, {"id", "payload", "pt"}, predicate,
                                  /*enable_predicate_filter=*/false));

    std::shared_ptr<arrow::DataType> result_type = arrow::struct_(
        {arrow::field("_VALUE_KIND", arrow::int8()), arrow::field("id", arrow::int64()),
         arrow::field("payload", arrow::utf8()), arrow::field("pt", arrow::utf8())});
    std::shared_ptr<arrow::Array> expected =
        arrow::ipc::internal::json::ArrayFromJSON(result_type, R"([
            [0, 0, null, "p0"],
            [0, 4, null, "p0"],
            [0, 5, "memory-value-5", "p0"]
        ])")
            .ValueOrDie();
    ASSERT_NE(nullptr, result);
    ASSERT_TRUE(std::make_shared<arrow::ChunkedArray>(expected)->Equals(*result))
        << result->ToString();
    ASSERT_OK(PrepareAndClose(writer.get()));
}

TEST_F(RealtimeWriteInteTest, TestUnionReadAfterColumnRename) {
    CreateTable(/*partition_keys=*/{});
    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> first_context, RealtimeContext::Create());
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> first_writer,
                         CreateRealtimeWriter(first_context));
    std::vector<Row> disk_rows = MakeRows(/*first_id=*/0, /*count=*/3, /*partition=*/"p0");
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> disk_batch,
                         MakeBatch(disk_rows, /*partitioned=*/false));
    ASSERT_OK(first_writer->Write(std::move(disk_batch)));
    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> disk_commits,
                         first_writer->PrepareCommitWithProgress(/*commit_identifier=*/0));
    ASSERT_OK(Commit(disk_commits, /*commit_identifier=*/0));
    ASSERT_OK(first_writer->Close());

    std::shared_ptr<arrow::Field> renamed_payload = arrow::field("renamed_payload", arrow::utf8());
    ASSERT_OK(TestHelper::WriteNextSchema(
        dir_->GetFileSystem(), table_path_,
        {DataField(0, fields_[0]), DataField(1, renamed_payload), DataField(2, fields_[2])},
        /*highest_field_id=*/2, options_));
    fields_[1] = renamed_payload;
    schema_ = arrow::schema(fields_);

    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> second_context,
                         RealtimeContext::Create());
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> second_writer,
                         CreateRealtimeWriter(second_context));
    std::vector<Row> memory_rows = MakeRows(/*first_id=*/3, /*count=*/2, /*partition=*/"p0");
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> memory_batch,
                         MakeBatch(memory_rows, /*partitioned=*/false));
    ASSERT_OK(second_writer->Write(std::move(memory_batch)));

    ASSERT_OK_AND_ASSIGN(std::shared_ptr<Plan> plan,
                         CreatePlan(second_context, /*predicate=*/nullptr));
    ASSERT_OK_AND_ASSIGN(std::shared_ptr<arrow::ChunkedArray> result,
                         ReadPlan(plan, second_context, {"id", "renamed_payload", "pt"},
                                  /*predicate=*/nullptr, /*enable_predicate_filter=*/false));
    std::shared_ptr<arrow::DataType> result_type = arrow::struct_(
        {arrow::field("_VALUE_KIND", arrow::int8()), arrow::field("id", arrow::int64()),
         renamed_payload, arrow::field("pt", arrow::utf8())});
    std::shared_ptr<arrow::Array> expected =
        arrow::ipc::internal::json::ArrayFromJSON(result_type, R"([
            [0, 0, "value-0", "p0"],
            [0, 1, "value-1", "p0"],
            [0, 2, "value-2", "p0"],
            [0, 3, "value-3", "p0"],
            [0, 4, "value-4", "p0"]
        ])")
            .ValueOrDie();
    ASSERT_NE(nullptr, result);
    ASSERT_TRUE(std::make_shared<arrow::ChunkedArray>(expected)->Equals(*result))
        << result->ToString();
    ASSERT_OK(PrepareAndClose(second_writer.get()));
}

TEST_F(RealtimeWriteInteTest, TestUnionReadWithNestedStructProjection) {
    std::shared_ptr<arrow::DataType> address_type =
        arrow::struct_({arrow::field("city", arrow::utf8()), arrow::field("zip", arrow::int64())});
    std::shared_ptr<arrow::DataType> profile_type = arrow::struct_(
        {arrow::field("name", arrow::utf8()), arrow::field("address", address_type)});
    fields_ = {arrow::field("id", arrow::int64()), arrow::field("profile", profile_type),
               arrow::field("pt", arrow::utf8())};
    schema_ = arrow::schema(fields_);
    CreateTable(/*partition_keys=*/{});

    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
                         RealtimeContext::Create());
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
                         CreateRealtimeWriter(realtime_context));
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> disk_batch,
                         MakeUnpartitionedBatchFromJson(R"([
                             [0, 0, ["disk-0", ["hangzhou", 310000]], "p0"],
                             [1, 1, ["disk-1", ["shanghai", 200000]], "p0"]
                         ])"));
    ASSERT_OK(writer->Write(std::move(disk_batch)));
    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> disk_commits,
                         writer->PrepareCommitWithProgress(/*commit_identifier=*/0));
    ASSERT_OK_AND_ASSIGN(int64_t disk_snapshot_id, Commit(disk_commits, /*commit_identifier=*/0));
    ASSERT_OK(writer->RefreshCommittedSnapshot(disk_snapshot_id));

    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> memory_batch,
                         MakeUnpartitionedBatchFromJson(R"([
                             [2, 2, ["memory-2", ["beijing", 100000]], "p0"],
                             [3, 3, ["memory-3", ["shenzhen", 518000]], "p0"]
                         ])"));
    ASSERT_OK(writer->Write(std::move(memory_batch)));

    ASSERT_OK_AND_ASSIGN(std::shared_ptr<Plan> plan,
                         CreatePlan(realtime_context, /*predicate=*/nullptr));
    std::shared_ptr<arrow::DataType> projected_address_type =
        arrow::struct_({arrow::field("city", arrow::utf8())});
    std::shared_ptr<arrow::DataType> projected_profile_type =
        arrow::struct_({arrow::field("address", projected_address_type)});
    std::shared_ptr<arrow::Schema> projected_schema = arrow::schema(
        {arrow::field("id", arrow::int64()), arrow::field("profile", projected_profile_type),
         arrow::field("pt", arrow::utf8())});
    ReadPlanWithSchemaAndCheck(plan, realtime_context, projected_schema, R"([
        [0, 0, [["hangzhou"]], "p0"],
        [0, 1, [["shanghai"]], "p0"],
        [0, 2, [["beijing"]], "p0"],
        [0, 3, [["shenzhen"]], "p0"]
    ])");
    ASSERT_OK(PrepareAndClose(writer.get()));
}

TEST_F(RealtimeWriteInteTest, TestUnionReadWithSelectedMapKeys) {
    auto run_test = [this](bool primary_key) {
        dir_ = UniqueTestDirectory::Create("local");
        ASSERT_NE(nullptr, dir_);
        table_path_ = PathUtil::JoinPath(dir_->Str(), "foo.db/bar");
        std::shared_ptr<arrow::DataType> map_type = arrow::map(arrow::utf8(), arrow::int64());
        fields_ = {arrow::field("id", arrow::int64()), arrow::field("tags", map_type),
                   arrow::field("pt", arrow::utf8())};
        schema_ = arrow::schema(fields_);
        options_["fields.tags.map.storage-layout"] = "shared-shredding";
        options_["fields.tags.map.shared-shredding.max-columns"] = "2";
        if (primary_key) {
            CreatePkTable();
        } else {
            CreateTable(/*partition_keys=*/{});
        }

        ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
                             RealtimeContext::Create());
        ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
                             CreateRealtimeWriter(realtime_context));
        ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> disk_batch,
                             MakeUnpartitionedBatchFromJson(R"([
                             [0, 0, [["a", 10], ["b", 20]], "p0"],
                             [1, 1, [["c", 30]], "p0"]
                         ])"));
        ASSERT_OK(writer->Write(std::move(disk_batch)));
        ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> disk_commits,
                             writer->PrepareCommitWithProgress(/*commit_identifier=*/0));
        ASSERT_OK_AND_ASSIGN(int64_t disk_snapshot_id,
                             Commit(disk_commits, /*commit_identifier=*/0));
        ASSERT_OK(writer->RefreshCommittedSnapshot(disk_snapshot_id));

        ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> memory_batch,
                             MakeUnpartitionedBatchFromJson(R"([
                             [2, 2, [["a", 40], ["c", 50]], "p0"],
                             [3, 3, null, "p0"]
                         ])"));
        ASSERT_OK(writer->Write(std::move(memory_batch)));
        ASSERT_OK_AND_ASSIGN(std::shared_ptr<Plan> plan,
                             CreatePlan(realtime_context, /*predicate=*/nullptr));

        std::shared_ptr<arrow::KeyValueMetadata> selected_keys =
            arrow::KeyValueMetadata::Make({DataField::MAP_SELECTED_KEYS}, {"c,a,missing"});
        std::shared_ptr<arrow::Field> selected_map_field = fields_[1]->WithMetadata(selected_keys);
        auto selected_map_schema = arrow::schema({fields_[0], selected_map_field, fields_[2]});
        ReadPlanWithSchemaAndCheck(plan, realtime_context, selected_map_schema, R"([
        [0, 0, [["a", 10]], "p0"],
        [0, 1, [["c", 30]], "p0"],
        [0, 2, [["c", 50], ["a", 40]], "p0"],
        [0, 3, null, "p0"]
    ])");

        ASSERT_OK_AND_ASSIGN(plan, CreatePlan(realtime_context, /*predicate=*/nullptr));
        auto c_map_field = std::make_unique<ArrowSchema>();
        ASSERT_TRUE(arrow::ExportField(*fields_[1], c_map_field.get()).ok());
        ASSERT_OK_AND_ASSIGN(std::unique_ptr<MapSharedShreddingAccessBuilder> access_builder,
                             MapSharedShreddingAccessBuilder::Create(c_map_field.get()));
        ASSERT_OK(access_builder->AddKey("a"));
        ASSERT_OK(access_builder->AddKey("missing"));
        ASSERT_OK_AND_ASSIGN(std::unique_ptr<ArrowSchema> c_access_field, access_builder->Build());
        auto access_field_result = arrow::ImportField(c_access_field.get());
        ASSERT_TRUE(access_field_result.ok()) << access_field_result.status().ToString();
        auto selected_struct_schema =
            arrow::schema({fields_[0], access_field_result.ValueOrDie(), fields_[2]});
        ReadPlanWithSchemaAndCheck(plan, realtime_context, selected_struct_schema, R"([
        [0, 0, [10, null], "p0"],
        [0, 1, [null, null], "p0"],
        [0, 2, [40, null], "p0"],
        [0, 3, null, "p0"]
    ])");
        ASSERT_OK(PrepareAndClose(writer.get()));
    };
    run_test(/*primary_key=*/false);
    ASSERT_FALSE(HasFatalFailure());
    run_test(/*primary_key=*/true);
    ASSERT_FALSE(HasFatalFailure());
}

TEST_F(RealtimeWriteInteTest, TestUnionReadWithVariantAccess) {
    auto run_test = [this](bool primary_key) {
        dir_ = UniqueTestDirectory::Create("local");
        ASSERT_NE(nullptr, dir_);
        table_path_ = PathUtil::JoinPath(dir_->Str(), "foo.db/bar");
        fields_ = {arrow::field("id", arrow::int32()), VariantTypeUtils::ToArrowField("v")};
        schema_ = arrow::schema(fields_);
        options_[Options::FILE_FORMAT] = "parquet";
        options_[Options::VARIANT_SHREDDING_SCHEMA] = R"({
        "type": "ROW",
        "fields": [{
            "id": 0,
            "name": "v",
            "type": {
                "type": "ROW",
                "fields": [
                    {"id": 1, "name": "age", "type": "BIGINT"},
                    {"id": 2, "name": "city", "type": "STRING"}
                ]
            }
        }]
    })";
        if (primary_key) {
            CreatePkTable();
        } else {
            CreateTable(/*partition_keys=*/{});
        }

        ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
                             RealtimeContext::Create());
        ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
                             CreateRealtimeWriter(realtime_context));
        ASSERT_OK_AND_ASSIGN(std::shared_ptr<arrow::StructArray> disk_data,
                             VariantTestData::BuildVariantBatch(
                                 fields_[0], fields_[1],
                                 {R"({"age":10,"city":"disk-a","note":"disk-fallback-a"})",
                                  R"({"age":20,"city":"disk-b","note":"disk-fallback-b"})"},
                                 pool_));
        std::shared_ptr<arrow::Array> disk_offsets =
            arrow::ipc::internal::json::ArrayFromJSON(arrow::int64(), "[0, 1]").ValueOrDie();
        arrow::ArrayVector disk_columns = {std::move(disk_offsets)};
        disk_columns.insert(disk_columns.end(), disk_data->fields().begin(),
                            disk_data->fields().end());
        ASSERT_OK_AND_ASSIGN(
            std::unique_ptr<RealtimeSchemaLayout> schema_layout,
            RealtimeSchemaLayout::Create(
                primary_key ? RealtimeStoreMode::PRIMARY_KEY : RealtimeStoreMode::APPEND_ONLY,
                schema_));
        const std::shared_ptr<arrow::Schema>& realtime_schema = schema_layout->InputSchema();
        std::shared_ptr<arrow::StructArray> disk_realtime_data =
            arrow::StructArray::Make(std::move(disk_columns), realtime_schema->fields())
                .ValueOrDie();
        ArrowArray disk_c_array;
        ASSERT_TRUE(arrow::ExportArray(*disk_realtime_data, &disk_c_array).ok());
        ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> disk_batch,
                             RecordBatchBuilder(&disk_c_array).SetBucket(0).Finish());
        ASSERT_OK(writer->Write(std::move(disk_batch)));
        ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> disk_commits,
                             writer->PrepareCommitWithProgress(/*commit_identifier=*/0));
        ASSERT_OK_AND_ASSIGN(int64_t disk_snapshot_id,
                             Commit(disk_commits, /*commit_identifier=*/0));
        ASSERT_OK(writer->RefreshCommittedSnapshot(disk_snapshot_id));

        ASSERT_OK_AND_ASSIGN(std::shared_ptr<arrow::StructArray> memory_data,
                             VariantTestData::BuildVariantBatch(
                                 fields_[0], fields_[1],
                                 {R"({"age":30,"city":"memory-a"})",
                                  R"({"age":40,"city":"memory-b","note":"memory-fallback-b"})"},
                                 pool_, /*id_offset=*/2));
        std::shared_ptr<arrow::Array> memory_offsets =
            arrow::ipc::internal::json::ArrayFromJSON(arrow::int64(), "[2, 3]").ValueOrDie();
        arrow::ArrayVector memory_columns = {std::move(memory_offsets)};
        memory_columns.insert(memory_columns.end(), memory_data->fields().begin(),
                              memory_data->fields().end());
        std::shared_ptr<arrow::StructArray> memory_realtime_data =
            arrow::StructArray::Make(std::move(memory_columns), realtime_schema->fields())
                .ValueOrDie();
        ArrowArray memory_c_array;
        ASSERT_TRUE(arrow::ExportArray(*memory_realtime_data, &memory_c_array).ok());
        ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> memory_batch,
                             RecordBatchBuilder(&memory_c_array).SetBucket(0).Finish());
        ASSERT_OK(writer->Write(std::move(memory_batch)));
        ASSERT_OK_AND_ASSIGN(std::shared_ptr<Plan> plan,
                             CreatePlan(realtime_context, /*predicate=*/nullptr));

        VariantAccessBuilder access_builder;
        auto age_target = std::make_unique<ArrowSchema>();
        auto city_target = std::make_unique<ArrowSchema>();
        auto note_target = std::make_unique<ArrowSchema>();
        ASSERT_TRUE(
            arrow::ExportField(*arrow::field("age", arrow::int64()), age_target.get()).ok());
        ASSERT_TRUE(
            arrow::ExportField(*arrow::field("city", arrow::utf8()), city_target.get()).ok());
        ASSERT_TRUE(
            arrow::ExportField(*arrow::field("note", arrow::utf8()), note_target.get()).ok());
        ASSERT_OK(access_builder.AddField(age_target.get(), "$.age", /*fail_on_error=*/false));
        ASSERT_OK(access_builder.AddField(city_target.get(), "$.city", /*fail_on_error=*/false));
        // `note` is intentionally absent from variant.shreddingSchema and must use binary fallback.
        ASSERT_OK(access_builder.AddField(note_target.get(), "$.note", /*fail_on_error=*/false));
        ASSERT_OK_AND_ASSIGN(std::unique_ptr<ArrowSchema> c_access_field,
                             access_builder.Build("v"));
        auto access_field_result = arrow::ImportField(c_access_field.get());
        ASSERT_TRUE(access_field_result.ok()) << access_field_result.status().ToString();
        auto access_schema = arrow::schema({fields_[0], access_field_result.ValueOrDie()});
        ReadPlanWithSchemaAndCheck(plan, realtime_context, access_schema, R"([
        [0, 0, [10, "disk-a", "disk-fallback-a"]],
        [0, 1, [20, "disk-b", "disk-fallback-b"]],
        [0, 2, [30, "memory-a", null]],
        [0, 3, [40, "memory-b", "memory-fallback-b"]]
    ])");
        ASSERT_OK(PrepareAndClose(writer.get()));
    };
    run_test(/*primary_key=*/false);
    ASSERT_FALSE(HasFatalFailure());
    run_test(/*primary_key=*/true);
    ASSERT_FALSE(HasFatalFailure());
}

TEST_F(RealtimeWriteInteTest, TestRefreshCommittedSnapshotReclaimsMemory) {
    CreateTable(/*partition_keys=*/{});
    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
                         RealtimeContext::Create());
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
                         CreateRealtimeWriter(realtime_context));

    std::vector<Row> disk_rows = MakeRows(/*first_id=*/0, /*count=*/10, /*partition=*/"p0");
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> disk_batch,
                         MakeBatch(disk_rows, /*partitioned=*/false));
    ASSERT_OK(writer->Write(std::move(disk_batch)));
    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> disk_commits,
                         writer->PrepareCommitWithProgress(/*commit_identifier=*/0));
    ASSERT_EQ(1, disk_commits.size());
    ASSERT_OK_AND_ASSIGN(int64_t committed_snapshot_id,
                         Commit(disk_commits, /*commit_identifier=*/0));

    std::vector<Row> memory_rows = MakeRows(/*first_id=*/10, /*count=*/2, /*partition=*/"p0");
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> memory_batch,
                         MakeBatch(memory_rows, /*partitioned=*/false));
    ASSERT_OK(writer->Write(std::move(memory_batch)));

    std::vector<Row> expected_rows = disk_rows;
    expected_rows.insert(expected_rows.end(), memory_rows.begin(), memory_rows.end());
    ASSERT_OK_AND_ASSIGN(std::vector<Row> read1, ReadRows(realtime_context));
    ASSERT_EQ(expected_rows, read1);
    ASSERT_OK_AND_ASSIGN(uint64_t memory_usage_before_refresh,
                         GetRealtimeMemoryUsage(realtime_context));

    ASSERT_OK(writer->RefreshCommittedSnapshot(committed_snapshot_id));

    ASSERT_OK_AND_ASSIGN(std::vector<Row> read2, ReadRows(realtime_context));
    ASSERT_EQ(read1, read2);
    ASSERT_OK_AND_ASSIGN(uint64_t memory_usage_after_refresh,
                         GetRealtimeMemoryUsage(realtime_context));
    ASSERT_LT(memory_usage_after_refresh, memory_usage_before_refresh);
    ASSERT_OK(PrepareAndClose(writer.get()));
}

TEST_F(RealtimeWriteInteTest, TestPlanPinsMemoryAcrossRefresh) {
    CreateTable(/*partition_keys=*/{});
    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
                         RealtimeContext::Create());
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
                         CreateRealtimeWriter(realtime_context));

    std::vector<Row> disk_rows = MakeRows(/*first_id=*/0, /*count=*/3, /*partition=*/"p0");
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> disk_batch,
                         MakeBatch(disk_rows, /*partitioned=*/false));
    ASSERT_OK(writer->Write(std::move(disk_batch)));
    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> disk_commits,
                         writer->PrepareCommitWithProgress(/*commit_identifier=*/0));
    ASSERT_OK_AND_ASSIGN(int64_t committed_snapshot_id,
                         Commit(disk_commits, /*commit_identifier=*/0));

    std::vector<Row> memory_rows = MakeRows(/*first_id=*/3, /*count=*/2, /*partition=*/"p0");
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> memory_batch,
                         MakeBatch(memory_rows, /*partitioned=*/false));
    ASSERT_OK(writer->Write(std::move(memory_batch)));
    ASSERT_OK_AND_ASSIGN(std::shared_ptr<Plan> pinned_plan,
                         CreatePlan(realtime_context, /*predicate=*/nullptr));

    ASSERT_OK(writer->RefreshCommittedSnapshot(committed_snapshot_id));

    std::vector<Row> expected_rows = disk_rows;
    expected_rows.insert(expected_rows.end(), memory_rows.begin(), memory_rows.end());
    ASSERT_OK_AND_ASSIGN(std::vector<Row> pinned_rows, ReadRows(pinned_plan, realtime_context));
    ASSERT_EQ(expected_rows, pinned_rows);

    ASSERT_OK_AND_ASSIGN(std::shared_ptr<Plan> refreshed_plan,
                         CreatePlan(realtime_context, /*predicate=*/nullptr));
    ASSERT_OK_AND_ASSIGN(std::vector<Row> refreshed_rows,
                         ReadRows(refreshed_plan, realtime_context));
    ASSERT_EQ(pinned_rows, refreshed_rows);
    ASSERT_OK(PrepareAndClose(writer.get()));
}

TEST_F(RealtimeWriteInteTest, TestReaderPinsMemoryAcrossRefresh) {
    CreateTable(/*partition_keys=*/{});
    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
                         RealtimeContext::Create());
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
                         CreateRealtimeWriter(realtime_context));

    std::vector<Row> rows = MakeRows(/*first_id=*/0, /*count=*/3, /*partition=*/"p0");
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> batch,
                         MakeBatch(rows, /*partitioned=*/false));
    ASSERT_OK(writer->Write(std::move(batch)));
    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> commits,
                         writer->PrepareCommitWithProgress(/*commit_identifier=*/0));
    ASSERT_EQ(1, commits.size());

    ASSERT_OK_AND_ASSIGN(std::shared_ptr<Plan> plan,
                         CreatePlan(realtime_context, /*predicate=*/nullptr));
    ASSERT_EQ(1, plan->Splits().size());
    std::shared_ptr<RealtimeSplit> realtime_split =
        std::dynamic_pointer_cast<RealtimeSplit>(plan->Splits()[0]);
    ASSERT_NE(nullptr, realtime_split);

    ReadContextBuilder read_builder(table_path_);
    read_builder.SetOptions(options_)
        .SetReadFieldNames({"id", "payload", "pt"})
        .WithRealtimeContext(realtime_context)
        .WithMemoryPool(pool_);
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<ReadContext> read_context, read_builder.Finish());
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<TableRead> table_read,
                         TableRead::Create(std::move(read_context)));
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<BatchReader> reader,
                         table_read->CreateReader(plan->Splits()));
    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContextImpl> realtime_context_impl,
                         RealtimeContextImpl::Cast(realtime_context));
    ASSERT_NOK_WITH_MSG(realtime_context_impl->ResolveReadView(realtime_split->OpaqueTicket()),
                        "ticket does not exist or has expired");
    ASSERT_OK_AND_ASSIGN(uint64_t memory_usage_before_refresh,
                         GetRealtimeMemoryUsage(realtime_context));
    ASSERT_GT(memory_usage_before_refresh, 0);

    ASSERT_OK_AND_ASSIGN(int64_t committed_snapshot_id, Commit(commits, /*commit_identifier=*/0));
    ASSERT_OK(writer->RefreshCommittedSnapshot(committed_snapshot_id));
    ASSERT_OK_AND_ASSIGN(uint64_t memory_usage_after_refresh,
                         GetRealtimeMemoryUsage(realtime_context));
    ASSERT_LT(memory_usage_after_refresh, memory_usage_before_refresh);

    ASSERT_OK_AND_ASSIGN(std::shared_ptr<arrow::ChunkedArray> result,
                         ReadResultCollector::CollectResult(std::move(reader)));
    std::shared_ptr<arrow::DataType> result_type = arrow::struct_(
        {arrow::field("_VALUE_KIND", arrow::int8()), arrow::field("id", arrow::int64()),
         arrow::field("payload", arrow::utf8()), arrow::field("pt", arrow::utf8())});
    std::shared_ptr<arrow::Array> expected =
        arrow::ipc::internal::json::ArrayFromJSON(result_type, R"([
            [0, 0, "value-0", "p0"],
            [0, 1, "value-1", "p0"],
            [0, 2, "value-2", "p0"]
        ])")
            .ValueOrDie();
    ASSERT_NE(nullptr, result);
    ASSERT_TRUE(std::make_shared<arrow::ChunkedArray>(expected)->Equals(*result))
        << result->ToString();
    ASSERT_OK(writer->Close());
}

class RealtimePrepareSpillInteTest
    : public RealtimeWriteInteTest,
      public ::testing::WithParamInterface<std::tuple<bool, bool, bool>> {};

TEST_P(RealtimePrepareSpillInteTest, TestPrepareFreezesWithoutSpilling) {
    const auto [primary_key, deletion_vectors, seal_before_prepare] = GetParam();
    const bool deduplicate = !primary_key && deletion_vectors;
    const bool merge_versions = primary_key || deduplicate;
    options_[Options::FILE_FORMAT] = "parquet";
    options_[Options::DELETION_VECTORS_ENABLED] = deletion_vectors ? "true" : "false";
    options_[Options::REALTIME_SPILL_ENABLED] = "true";
    if (primary_key) {
        CreatePkTable();
    } else if (deduplicate) {
        ConfigureRealtimeDeduplicateTable();
    } else {
        CreateTable(/*partition_keys=*/{});
    }
    const std::string temp_directory = PathUtil::JoinPath(dir_->Str(), "prepare-without-spill");
    ASSERT_OK(dir_->GetFileSystem()->Mkdirs(temp_directory));
    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> context, RealtimeContext::Create());
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
                         CreateRealtimeWriter(context, temp_directory));
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> initial,
                         MakeBatch({{1, "old", "p0"}, {2, "two", "p0"}},
                                   /*partitioned=*/false));
    ASSERT_OK(writer->Write(std::move(initial)));
    if (seal_before_prepare) {
        ASSERT_OK(writer->Seal());
    }
    const int64_t expected_spill_files = seal_before_prepare ? 1 : 0;
    ASSERT_EQ(expected_spill_files,
              TestHelper::CountChannelFiles(dir_->GetFileSystem(), temp_directory));

    const std::vector<Row> next_rows =
        merge_versions ? std::vector<Row>{{1, "new", "p0"}, {3, "three", "p0"}}
                       : std::vector<Row>{{3, "three", "p0"}, {4, "four", "p0"}};
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> next,
                         MakeBatch(next_rows, /*partitioned=*/false));
    ASSERT_OK(writer->Write(std::move(next)));
    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> progress,
                         writer->PrepareCommitWithProgress(/*commit_identifier=*/0));
    ASSERT_EQ(1, progress.size());
    ASSERT_EQ(OffsetRange(0, 4), progress[0].offset_range);
    if (deduplicate) {
        ASSERT_NE(nullptr, progress[0].deleted_offsets);
        ASSERT_EQ("{0}", progress[0].deleted_offsets->ToString());
    }
    ASSERT_EQ(expected_spill_files,
              TestHelper::CountChannelFiles(dir_->GetFileSystem(), temp_directory));
    ASSERT_OK_AND_ASSIGN(RealtimeMetricValues usage, ReadRealtimeMetrics(writer->GetMetrics()));
    ASSERT_EQ(0, usage.building_rows);
    ASSERT_EQ(4, usage.sealed_rows);
    ASSERT_GT(usage.sealed_memory, 0);
    const std::vector<Row> expected =
        merge_versions
            ? std::vector<Row>{{1, "new", "p0"}, {2, "two", "p0"}, {3, "three", "p0"}}
            : std::vector<Row>{
                  {1, "old", "p0"}, {2, "two", "p0"}, {3, "three", "p0"}, {4, "four", "p0"}};
    ASSERT_OK_AND_ASSIGN(std::vector<Row> visible, ReadRows(context));
    std::sort(visible.begin(), visible.end());
    ASSERT_EQ(expected, visible);
    ASSERT_OK(writer->Seal());
    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> empty,
                         writer->PrepareCommitWithProgress(/*commit_identifier=*/1));
    ASSERT_TRUE(empty.empty());
    ASSERT_EQ(expected_spill_files,
              TestHelper::CountChannelFiles(dir_->GetFileSystem(), temp_directory));

    ASSERT_OK_AND_ASSIGN(int64_t snapshot, Commit(progress, /*commit_identifier=*/1));
    std::shared_ptr<RealtimeContext> disk_context;
    if (primary_key && deletion_vectors) {
        // Ordinary PK DV scans skip L0; an empty context reads it without any memory tail.
        ASSERT_OK_AND_ASSIGN(disk_context, RealtimeContext::Create());
    }
    ASSERT_OK_AND_ASSIGN(visible, ReadRows(disk_context));
    std::sort(visible.begin(), visible.end());
    ASSERT_EQ(expected, visible);
    ASSERT_OK(writer->RefreshCommittedSnapshot(snapshot));
    ASSERT_TRUE(WaitForChannelFileCount(temp_directory, /*expected_count=*/0));
    ASSERT_OK_AND_ASSIGN(usage, ReadRealtimeMetrics(writer->GetMetrics()));
    ASSERT_EQ(0, usage.sealed_memory);
    ASSERT_EQ(0, usage.total_rows);
    ASSERT_OK_AND_ASSIGN(visible, ReadRows(context));
    std::sort(visible.begin(), visible.end());
    ASSERT_EQ(expected, visible);
    ASSERT_OK(writer->Close());
}

INSTANTIATE_TEST_SUITE_P(Prepare, RealtimePrepareSpillInteTest,
                         ::testing::Combine(::testing::Bool(), ::testing::Bool(),
                                            ::testing::Bool()));

TEST_F(RealtimeWriteInteTest, TestAppendRealtimeSpillFileLifecycle) {
    options_[Options::REALTIME_SPILL_ENABLED] = "true";
    CreateTable(/*partition_keys=*/{});
    const std::string temp_directory = PathUtil::JoinPath(dir_->Str(), "append-realtime-spill");
    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
                         RealtimeContext::Create());
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
                         CreateRealtimeWriter(realtime_context, temp_directory));

    std::vector<Row> first_rows = MakeRows(/*first_id=*/0, /*count=*/4, /*partition=*/"p0");
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> first_batch,
                         MakeBatch(std::vector<Row>(first_rows.begin(), first_rows.begin() + 2),
                                   /*partitioned=*/false));
    ASSERT_OK(writer->Write(std::move(first_batch)));
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> second_batch,
                         MakeBatch(std::vector<Row>(first_rows.begin() + 2, first_rows.end()),
                                   /*partitioned=*/false));
    ASSERT_OK(writer->Write(std::move(second_batch)));
    ASSERT_OK_AND_ASSIGN(RealtimeMetricValues memory_metrics_before_spill,
                         ReadRealtimeMetrics(writer->GetMetrics()));
    ASSERT_GT(memory_metrics_before_spill.building_memory, 0);
    ASSERT_EQ(0, memory_metrics_before_spill.sealed_memory);
    ASSERT_EQ(memory_metrics_before_spill.building_memory,
              memory_metrics_before_spill.total_memory);

    ASSERT_OK(writer->Seal());
    ASSERT_EQ(1, TestHelper::CountChannelFiles(dir_->GetFileSystem(), temp_directory));
    ASSERT_OK_AND_ASSIGN(RealtimeMetricValues memory_metrics_after_spill,
                         ReadRealtimeMetrics(writer->GetMetrics()));
    ASSERT_EQ(0, memory_metrics_after_spill.building_memory);
    ASSERT_EQ(0, memory_metrics_after_spill.sealed_memory);
    ASSERT_EQ(0, memory_metrics_after_spill.total_memory);
    ASSERT_EQ(0, memory_metrics_after_spill.building_rows);
    ASSERT_EQ(4, memory_metrics_after_spill.sealed_rows);
    ASSERT_EQ(4, memory_metrics_after_spill.total_rows);

    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> first_progress,
                         writer->PrepareCommitWithProgress(/*commit_identifier=*/0));
    ASSERT_EQ(1, first_progress.size());
    ASSERT_OK_AND_ASSIGN(std::shared_ptr<Plan> pinned_plan,
                         CreatePlan(realtime_context, /*predicate=*/nullptr));
    ASSERT_EQ(1, pinned_plan->Splits().size());
    ASSERT_NE(nullptr, std::dynamic_pointer_cast<RealtimeSplit>(pinned_plan->Splits()[0]));
    ASSERT_OK_AND_ASSIGN(int64_t first_snapshot_id,
                         Commit(first_progress, /*commit_identifier=*/0));

    ASSERT_OK(writer->RefreshCommittedSnapshot(first_snapshot_id));
    ASSERT_EQ(1, TestHelper::CountChannelFiles(dir_->GetFileSystem(), temp_directory));
    ASSERT_OK_AND_ASSIGN(RealtimeMetricValues memory_metrics_after_reclaim,
                         ReadRealtimeMetrics(writer->GetMetrics()));
    ASSERT_EQ(0, memory_metrics_after_reclaim.total_memory);
    ASSERT_EQ(0, memory_metrics_after_reclaim.total_rows);
    ASSERT_OK_AND_ASSIGN(std::vector<Row> pinned_rows, ReadRows(pinned_plan, realtime_context));
    ASSERT_EQ(first_rows, pinned_rows);
    ASSERT_TRUE(WaitForChannelFileCount(temp_directory, /*expected_count=*/0));

    std::vector<Row> second_rows = MakeRows(/*first_id=*/4, /*count=*/4, /*partition=*/"p0");
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> third_batch,
                         MakeBatch(std::vector<Row>(second_rows.begin(), second_rows.begin() + 2),
                                   /*partitioned=*/false));
    ASSERT_OK(writer->Write(std::move(third_batch)));
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> fourth_batch,
                         MakeBatch(std::vector<Row>(second_rows.begin() + 2, second_rows.end()),
                                   /*partitioned=*/false));
    ASSERT_OK(writer->Write(std::move(fourth_batch)));
    ASSERT_OK(writer->Seal());
    ASSERT_EQ(1, TestHelper::CountChannelFiles(dir_->GetFileSystem(), temp_directory));

    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> second_progress,
                         writer->PrepareCommitWithProgress(/*commit_identifier=*/1));
    ASSERT_EQ(1, second_progress.size());
    ASSERT_OK_AND_ASSIGN(int64_t second_snapshot_id,
                         Commit(second_progress, /*commit_identifier=*/1));
    ASSERT_OK(writer->RefreshCommittedSnapshot(second_snapshot_id));
    ASSERT_EQ(0, TestHelper::CountChannelFiles(dir_->GetFileSystem(), temp_directory));

    std::vector<Row> expected_rows = first_rows;
    expected_rows.insert(expected_rows.end(), second_rows.begin(), second_rows.end());
    ASSERT_OK_AND_ASSIGN(std::vector<Row> actual_rows, ReadRows(realtime_context));
    ASSERT_EQ(expected_rows, actual_rows);
    ASSERT_OK(writer->Close());
}

TEST_F(RealtimeWriteInteTest, TestDeduplicateRealtimeSpillFileLifecycle) {
    options_[Options::REALTIME_SPILL_ENABLED] = "true";
    ConfigureRealtimeDeduplicateTable();
    const std::string temp_directory = PathUtil::JoinPath(dir_->Str(), "deduplicate-spill");
    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> context, RealtimeContext::Create());
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
                         CreateRealtimeWriter(context, temp_directory));
    const std::vector<Row> initial_rows = {{1, "old", "p0"}, {2, "two", "p0"}};
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> initial,
                         MakeBatch(initial_rows, /*partitioned=*/false));
    ASSERT_OK(writer->Write(std::move(initial)));
    ASSERT_OK_AND_ASSIGN(std::shared_ptr<Plan> pinned, CreatePlan(context, /*predicate=*/nullptr));
    ASSERT_OK(writer->Seal());
    ASSERT_EQ(1, TestHelper::CountChannelFiles(dir_->GetFileSystem(), temp_directory));
    ASSERT_OK_AND_ASSIGN(RealtimeMetricValues sealed_usage,
                         ReadRealtimeMetrics(writer->GetMetrics()));
    // Arrow payloads have spilled, but the sealed key index still occupies memory.
    ASSERT_GT(sealed_usage.sealed_memory, 0);
    ASSERT_GT(sealed_usage.building_memory, 0);
    ASSERT_EQ(2, sealed_usage.sealed_rows);
    ASSERT_EQ(0, sealed_usage.building_rows);

    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> updates,
                         MakeBatch({{1, "new", "p0"}, {2, "deleted", "p0"}, {3, "three", "p0"}},
                                   /*partitioned=*/false, /*bucket=*/0,
                                   {RecordBatch::RowKind::INSERT, RecordBatch::RowKind::DELETE,
                                    RecordBatch::RowKind::INSERT}));
    ASSERT_OK(writer->Write(std::move(updates)));
    const std::vector<Row> expected = {{1, "new", "p0"}, {3, "three", "p0"}};
    ASSERT_OK_AND_ASSIGN(std::vector<Row> visible, ReadRows(context));
    ASSERT_EQ(expected, visible);
    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> progress,
                         writer->PrepareCommitWithProgress(/*commit_identifier=*/0));
    ASSERT_EQ(1, progress.size());
    ASSERT_EQ("{0,1}", progress[0].deleted_offsets->ToString());
    ASSERT_OK_AND_ASSIGN(int64_t snapshot, Commit(progress, /*commit_identifier=*/0));
    ASSERT_OK(writer->RefreshCommittedSnapshot(snapshot));
    ASSERT_OK_AND_ASSIGN(visible, ReadRows(context));
    ASSERT_EQ(expected, visible);
    ASSERT_OK_AND_ASSIGN(RealtimeMetricValues reclaimed_usage,
                         ReadRealtimeMetrics(writer->GetMetrics()));
    ASSERT_EQ(0, reclaimed_usage.sealed_memory);
    ASSERT_EQ(0, reclaimed_usage.total_rows);
    // The current, empty building key index retains its allocated buckets.
    ASSERT_GT(reclaimed_usage.building_memory, 0);
    ASSERT_EQ(reclaimed_usage.building_memory, reclaimed_usage.total_memory);
    ASSERT_OK_AND_ASSIGN(visible, ReadRows(pinned, context));
    ASSERT_EQ(initial_rows, visible);
    ASSERT_TRUE(WaitForChannelFileCount(temp_directory, /*expected_count=*/0));
    ASSERT_OK(writer->Close());
}

TEST_F(RealtimeWriteInteTest, TestPkDvRealtimeSpillFileLifecycle) {
    options_[Options::FILE_FORMAT] = "parquet";
    options_[Options::DELETION_VECTORS_ENABLED] = "true";
    options_[Options::REALTIME_SPILL_ENABLED] = "true";
    CreatePkTable();
    const std::string temp_directory = PathUtil::JoinPath(dir_->Str(), "pk-dv-realtime-spill");
    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
                         RealtimeContext::Create());
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
                         CreateRealtimeWriter(realtime_context, temp_directory));

    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> first_batch,
                         MakeBatch({Row{1, "old-one", "p0"}, Row{2, "two", "p0"}},
                                   /*partitioned=*/false, /*bucket=*/0,
                                   {RecordBatch::RowKind::INSERT, RecordBatch::RowKind::INSERT}));
    ASSERT_OK(writer->Write(std::move(first_batch)));
    ASSERT_OK_AND_ASSIGN(
        std::unique_ptr<RecordBatch> second_batch,
        MakeBatch({Row{1, "new-one", "p0"}, Row{3, "three", "p0"}},
                  /*partitioned=*/false, /*bucket=*/0,
                  {RecordBatch::RowKind::UPDATE_AFTER, RecordBatch::RowKind::INSERT}));
    ASSERT_OK(writer->Write(std::move(second_batch)));
    ASSERT_OK(writer->Seal());
    ASSERT_EQ(1, TestHelper::CountChannelFiles(dir_->GetFileSystem(), temp_directory));

    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> first_progress,
                         writer->PrepareCommitWithProgress(/*commit_identifier=*/0));
    ASSERT_EQ(1, first_progress.size());
    ASSERT_OK_AND_ASSIGN(std::shared_ptr<Plan> pinned_plan,
                         CreatePlan(realtime_context, /*predicate=*/nullptr));
    ASSERT_EQ(1, pinned_plan->Splits().size());
    ASSERT_NE(nullptr, std::dynamic_pointer_cast<RealtimeSplit>(pinned_plan->Splits()[0]));
    ASSERT_OK_AND_ASSIGN(int64_t first_snapshot_id,
                         Commit(first_progress, /*commit_identifier=*/0));

    ASSERT_OK(writer->RefreshCommittedSnapshot(first_snapshot_id));
    ASSERT_EQ(1, TestHelper::CountChannelFiles(dir_->GetFileSystem(), temp_directory));
    ASSERT_OK_AND_ASSIGN(std::vector<Row> pinned_rows, ReadRows(pinned_plan, realtime_context));
    ASSERT_EQ((std::vector<Row>{{1, "new-one", "p0"}, {2, "two", "p0"}, {3, "three", "p0"}}),
              pinned_rows);
    ASSERT_TRUE(WaitForChannelFileCount(temp_directory, /*expected_count=*/0));

    ASSERT_OK_AND_ASSIGN(Snapshot compact_snapshot, CompactAndCommit(/*partition=*/{}, /*bucket=*/0,
                                                                     /*commit_identifier=*/1));
    ASSERT_OK(writer->RefreshCommittedSnapshot(compact_snapshot.Id()));

    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> third_batch,
                         MakeBatch({Row{2, "latest-two", "p0"}}, /*partitioned=*/false,
                                   /*bucket=*/0, {RecordBatch::RowKind::UPDATE_AFTER}));
    ASSERT_OK(writer->Write(std::move(third_batch)));
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> fourth_batch,
                         MakeBatch({Row{3, "deleted-three", "p0"}, Row{4, "four", "p0"}},
                                   /*partitioned=*/false, /*bucket=*/0,
                                   {RecordBatch::RowKind::DELETE, RecordBatch::RowKind::INSERT}));
    ASSERT_OK(writer->Write(std::move(fourth_batch)));
    ASSERT_OK(writer->Seal());
    ASSERT_EQ(1, TestHelper::CountChannelFiles(dir_->GetFileSystem(), temp_directory));

    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> second_progress,
                         writer->PrepareCommitWithProgress(/*commit_identifier=*/2));
    ASSERT_EQ(1, second_progress.size());
    ASSERT_OK_AND_ASSIGN(int64_t second_snapshot_id,
                         Commit(second_progress, /*commit_identifier=*/2));

    const std::vector<Row> expected_rows = {
        {1, "new-one", "p0"}, {2, "latest-two", "p0"}, {4, "four", "p0"}};
    ASSERT_OK_AND_ASSIGN(std::vector<Row> rows_before_reclaim, ReadRows(realtime_context));
    ASSERT_EQ(expected_rows, rows_before_reclaim);

    ASSERT_OK(writer->RefreshCommittedSnapshot(second_snapshot_id));
    ASSERT_EQ(0, TestHelper::CountChannelFiles(dir_->GetFileSystem(), temp_directory));

    ASSERT_OK_AND_ASSIGN(std::vector<Row> actual_rows, ReadRows(realtime_context));
    ASSERT_EQ(expected_rows, actual_rows);
    ASSERT_OK(writer->Close());
}

TEST_F(RealtimeWriteInteTest, TestRepeatedCommitReadAndRefresh) {
    CreateTable(/*partition_keys=*/{});
    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
                         RealtimeContext::Create());
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
                         CreateRealtimeWriter(realtime_context));
    constexpr int64_t kRoundCount = 3;
    constexpr int64_t kRowsPerRound = 4;
    std::vector<Row> expected_rows;
    for (int64_t round = 0; round < kRoundCount; ++round) {
        std::vector<Row> rows = MakeRows(round * kRowsPerRound, kRowsPerRound, /*partition=*/"p0");
        ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> batch,
                             MakeBatch(rows, /*partitioned=*/false));
        ASSERT_OK(writer->Write(std::move(batch)));
        ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> commits,
                             writer->PrepareCommitWithProgress(/*commit_identifier=*/round));
        ASSERT_EQ(1, commits.size());
        ASSERT_EQ(OffsetRange(round * kRowsPerRound, (round + 1) * kRowsPerRound),
                  commits[0].offset_range);
        ASSERT_OK_AND_ASSIGN(int64_t committed_snapshot_id,
                             Commit(commits, /*commit_identifier=*/round));
        expected_rows.insert(expected_rows.end(), rows.begin(), rows.end());

        ASSERT_OK_AND_ASSIGN(std::vector<Row> read_before_refresh, ReadRows(realtime_context));
        ASSERT_EQ(expected_rows, read_before_refresh);
        ASSERT_OK_AND_ASSIGN(uint64_t memory_usage_before_refresh,
                             GetRealtimeMemoryUsage(realtime_context));
        ASSERT_GT(memory_usage_before_refresh, 0);

        ASSERT_OK(writer->RefreshCommittedSnapshot(committed_snapshot_id));

        ASSERT_OK_AND_ASSIGN(std::vector<Row> read_after_refresh, ReadRows(realtime_context));
        ASSERT_EQ(read_before_refresh, read_after_refresh);
        ASSERT_OK_AND_ASSIGN(uint64_t memory_usage_after_refresh,
                             GetRealtimeMemoryUsage(realtime_context));
        ASSERT_EQ(0, memory_usage_after_refresh);
    }
    ASSERT_OK(writer->Close());
}

TEST_F(RealtimeWriteInteTest, TestRefreshLatestSnapshotReclaimsMultipleCommittedSegments) {
    CreateTable(/*partition_keys=*/{});
    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
                         RealtimeContext::Create());
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
                         CreateRealtimeWriter(realtime_context));
    constexpr int64_t kSnapshotCount = 3;
    constexpr int64_t kRowsPerSnapshot = 2;
    std::vector<Row> expected_rows;
    int64_t latest_snapshot_id = -1;
    for (int64_t snapshot_index = 0; snapshot_index < kSnapshotCount; ++snapshot_index) {
        std::vector<Row> rows =
            MakeRows(snapshot_index * kRowsPerSnapshot, kRowsPerSnapshot, /*partition=*/"p0");
        ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> batch,
                             MakeBatch(rows, /*partitioned=*/false));
        ASSERT_OK(writer->Write(std::move(batch)));
        ASSERT_OK_AND_ASSIGN(
            std::vector<RealtimeCommitProgress> commits,
            writer->PrepareCommitWithProgress(/*commit_identifier=*/snapshot_index));
        ASSERT_EQ(1, commits.size());
        ASSERT_OK_AND_ASSIGN(latest_snapshot_id,
                             Commit(commits, /*commit_identifier=*/snapshot_index));
        expected_rows.insert(expected_rows.end(), rows.begin(), rows.end());
    }

    ASSERT_OK_AND_ASSIGN(std::vector<Row> rows_before_refresh, ReadRows(realtime_context));
    ASSERT_EQ(expected_rows, rows_before_refresh);
    ASSERT_OK_AND_ASSIGN(uint64_t memory_usage_before_refresh,
                         GetRealtimeMemoryUsage(realtime_context));
    ASSERT_GT(memory_usage_before_refresh, 0);

    ASSERT_OK(writer->RefreshCommittedSnapshot(latest_snapshot_id));

    ASSERT_OK_AND_ASSIGN(std::vector<Row> rows_after_refresh, ReadRows(realtime_context));
    ASSERT_EQ(rows_before_refresh, rows_after_refresh);
    ASSERT_OK_AND_ASSIGN(uint64_t memory_usage_after_refresh,
                         GetRealtimeMemoryUsage(realtime_context));
    ASSERT_EQ(0, memory_usage_after_refresh);
    ASSERT_OK(writer->Close());
}

TEST_F(RealtimeWriteInteTest, TestOverwriteRequiresReopenRealtimeContext) {
    CreateTable(/*partition_keys=*/{});
    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
                         RealtimeContext::Create());
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
                         CreateRealtimeWriter(realtime_context));

    std::vector<Row> committed_rows = MakeRows(/*first_id=*/0, /*count=*/3, /*partition=*/"p0");
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> committed_batch,
                         MakeBatch(committed_rows, /*partitioned=*/false));
    ASSERT_OK(writer->Write(std::move(committed_batch)));
    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> commits,
                         writer->PrepareCommitWithProgress(/*commit_identifier=*/0));
    ASSERT_OK_AND_ASSIGN(int64_t committed_snapshot_id, Commit(commits, /*commit_identifier=*/0));
    ASSERT_OK(writer->RefreshCommittedSnapshot(committed_snapshot_id));

    std::vector<Row> building_rows = MakeRows(/*first_id=*/3, /*count=*/2, /*partition=*/"p0");
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> building_batch,
                         MakeBatch(building_rows, /*partitioned=*/false));
    ASSERT_OK(writer->Write(std::move(building_batch)));
    ASSERT_OK_AND_ASSIGN(uint64_t memory_usage_before_overwrite,
                         GetRealtimeMemoryUsage(realtime_context));
    ASSERT_GT(memory_usage_before_overwrite, 0);

    ASSERT_OK_AND_ASSIGN(int64_t overwrite_snapshot_id, TruncateTable(/*commit_identifier=*/1));
    ASSERT_OK_AND_ASSIGN(CoreOptions core_options, CoreOptions::FromMap(options_));
    SnapshotManager snapshot_manager(core_options.GetFileSystem(), table_path_);
    ASSERT_OK_AND_ASSIGN(Snapshot overwrite_snapshot,
                         snapshot_manager.LoadSnapshot(overwrite_snapshot_id));
    ASSERT_EQ(Snapshot::CommitKind::Overwrite(), overwrite_snapshot.GetCommitKind());
    ASSERT_FALSE(RealtimeCommitProperties::GetOffsetsPath(overwrite_snapshot));

    ASSERT_NOK_WITH_MSG(writer->RefreshCommittedSnapshot(overwrite_snapshot_id),
                        "recreate RealtimeContext");
    ASSERT_OK_AND_ASSIGN(uint64_t memory_usage_after_failed_refresh,
                         GetRealtimeMemoryUsage(realtime_context));
    ASSERT_EQ(memory_usage_before_overwrite, memory_usage_after_failed_refresh);
    ASSERT_NOK_WITH_MSG(writer->Close(), "not covered by a successful");
    writer.reset();
    realtime_context.reset();

    ASSERT_OK_AND_ASSIGN(realtime_context, RealtimeContext::Create());
    ASSERT_OK_AND_ASSIGN(writer, CreateRealtimeWriter(realtime_context));
    ResetExternalOffset(/*partition=*/{}, /*bucket=*/0, /*next_offset=*/0);
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> replay_batch,
                         MakeBatch(building_rows, /*partitioned=*/false));
    ASSERT_OK(writer->Write(std::move(replay_batch)));
    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> replay_commits,
                         writer->PrepareCommitWithProgress(/*commit_identifier=*/2));
    ASSERT_EQ(1, replay_commits.size());
    ASSERT_EQ(OffsetRange(0, 2), replay_commits[0].offset_range);
    ASSERT_OK_AND_ASSIGN(int64_t replay_snapshot_id,
                         Commit(replay_commits, /*commit_identifier=*/2));
    ASSERT_OK(writer->RefreshCommittedSnapshot(replay_snapshot_id));
    ASSERT_OK_AND_ASSIGN(std::vector<Row> replayed_rows, ReadRows(realtime_context));
    ASSERT_EQ(building_rows, replayed_rows);
    ASSERT_OK(writer->Close());
}

TEST_F(RealtimeWriteInteTest, TestReopenRealtimeContextAfterRollback) {
    CreateTable(/*partition_keys=*/{});
    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
                         RealtimeContext::Create());
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
                         CreateRealtimeWriter(realtime_context));

    std::vector<Row> first_rows = MakeRows(/*first_id=*/0, /*count=*/3, /*partition=*/"p0");
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> first_batch,
                         MakeBatch(first_rows, /*partitioned=*/false));
    ASSERT_OK(writer->Write(std::move(first_batch)));
    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> first_commits,
                         writer->PrepareCommitWithProgress(/*commit_identifier=*/0));
    ASSERT_OK_AND_ASSIGN(int64_t first_snapshot_id, Commit(first_commits, /*commit_identifier=*/0));
    ASSERT_OK(writer->RefreshCommittedSnapshot(first_snapshot_id));

    std::vector<Row> second_rows = MakeRows(/*first_id=*/3, /*count=*/2, /*partition=*/"p0");
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> second_batch,
                         MakeBatch(second_rows, /*partitioned=*/false));
    ASSERT_OK(writer->Write(std::move(second_batch)));
    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> second_commits,
                         writer->PrepareCommitWithProgress(/*commit_identifier=*/1));
    ASSERT_OK_AND_ASSIGN(int64_t second_snapshot_id,
                         Commit(second_commits, /*commit_identifier=*/1));
    ASSERT_OK(writer->RefreshCommittedSnapshot(second_snapshot_id));

    ASSERT_OK_AND_ASSIGN(int64_t rollback_snapshot_id, RollbackToAsLatest(first_snapshot_id));
    ASSERT_OK_AND_ASSIGN(RealtimeOffsetMap rollback_offsets, ReadCommittedOffsets());
    ASSERT_EQ(1, rollback_offsets.size());
    ASSERT_EQ(3, rollback_offsets.at(RealtimePartitionBucket(/*partition=*/{}, /*bucket=*/0)));
    ASSERT_NOK_WITH_MSG(writer->RefreshCommittedSnapshot(rollback_snapshot_id),
                        "recreate RealtimeContext");
    ASSERT_OK(writer->Close());
    writer.reset();
    realtime_context.reset();

    // Reopen the same real-time writer identity from the target snapshot's progress and replay
    // input after that restored boundary.
    ASSERT_OK_AND_ASSIGN(realtime_context, RealtimeContext::Create());
    ASSERT_OK_AND_ASSIGN(writer, CreateRealtimeWriter(realtime_context));
    ResetExternalOffset(/*partition=*/{}, /*bucket=*/0, /*next_offset=*/3);
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> replay_batch,
                         MakeBatch(second_rows, /*partitioned=*/false));
    ASSERT_OK(writer->Write(std::move(replay_batch)));
    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> replay_commits,
                         writer->PrepareCommitWithProgress(/*commit_identifier=*/2));
    ASSERT_EQ(1, replay_commits.size());
    ASSERT_EQ(OffsetRange(3, 5), replay_commits[0].offset_range);
    ASSERT_OK(Commit(replay_commits, /*commit_identifier=*/2));

    std::vector<Row> expected_rows = first_rows;
    expected_rows.insert(expected_rows.end(), second_rows.begin(), second_rows.end());
    ASSERT_OK_AND_ASSIGN(std::vector<Row> actual_rows, ReadRows());
    ASSERT_EQ(expected_rows, actual_rows);
    ASSERT_OK(writer->Close());
}

TEST_F(RealtimeWriteInteTest, TestConcurrentWritePrepareCommitReadAndRefresh) {
    auto run_test = [this](bool primary_key) {
        dir_ = UniqueTestDirectory::Create("local");
        ASSERT_NE(nullptr, dir_);
        table_path_ = PathUtil::JoinPath(dir_->Str(), "foo.db/bar");
        ResetExternalOffset(/*partition=*/{}, /*bucket=*/0, /*next_offset=*/0);
        if (primary_key) {
            CreatePkTable();
        } else {
            CreateTable(/*partition_keys=*/{});
        }
        ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
                             RealtimeContext::Create());
        ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
                             CreateRealtimeWriter(realtime_context));
        constexpr int32_t kPrepareThreadCount = 4;
        constexpr int32_t kReadThreadCount = 4;
        constexpr int64_t kBatchCount = 12;
        constexpr int64_t kRowsPerBatch = 2;
        const int64_t total_rows = kBatchCount * (primary_key ? 3 : kRowsPerBatch);

        std::vector<std::vector<Row>> pk_batches;
        std::vector<std::vector<RecordBatch::RowKind>> pk_row_kinds;
        std::vector<std::vector<Row>> pk_expected_states(1);
        if (primary_key) {
            std::map<int64_t, Row> current_rows;
            for (int64_t batch_index = 0; batch_index < kBatchCount; ++batch_index) {
                const int64_t key = batch_index % 4;
                const int64_t deleted_key = (key + 2) % 4;
                std::vector<Row> rows = {
                    {key, "update-" + std::to_string(batch_index), "p0"},
                    {key, "latest-" + std::to_string(batch_index), "p0"},
                    {deleted_key, "deleted-" + std::to_string(batch_index), "p0"}};
                pk_batches.push_back(rows);
                pk_row_kinds.push_back({batch_index < 4 ? RecordBatch::RowKind::INSERT
                                                        : RecordBatch::RowKind::UPDATE_AFTER,
                                        RecordBatch::RowKind::UPDATE_AFTER,
                                        RecordBatch::RowKind::DELETE});
                current_rows[key] = rows[1];
                current_rows.erase(deleted_key);
                std::vector<Row> expected;
                for (const auto& [id, row] : current_rows) {
                    static_cast<void>(id);
                    expected.push_back(row);
                }
                pk_expected_states.push_back(std::move(expected));
            }
        }

        auto validate_read = [&](const std::vector<Row>& rows) {
            if (!primary_key) {
                return ValidateReadPrefix(rows, total_rows);
            }
            if (std::find(pk_expected_states.begin(), pk_expected_states.end(), rows) ==
                pk_expected_states.end()) {
                return Status::Invalid("PK real-time read does not match any completed write");
            }
            return Status::OK();
        };

        std::atomic<bool> writer_done{false};
        std::atomic<bool> prepare_done{false};
        std::atomic<bool> commit_done{false};
        std::atomic<bool> refresh_done{false};
        std::atomic<int64_t> next_prepare_identifier{0};
        std::atomic<int32_t> commit_count{0};
        std::atomic<int32_t> refresh_count{0};
        ConcurrentTestState state;
        std::map<int64_t, RealtimeCommitProgress> pending_commits;
        std::deque<int64_t> pending_snapshot_ids;
        std::vector<int32_t> prepare_call_counts(kPrepareThreadCount, 0);
        std::vector<int32_t> read_call_counts(kReadThreadCount, 0);

        auto enqueue_prepared_commits = [&](std::vector<RealtimeCommitProgress>&& commits) {
            std::string error;
            {
                std::lock_guard<std::mutex> lock(state.mutex);
                for (RealtimeCommitProgress& commit : commits) {
                    int64_t offset_begin = commit.offset_range.begin;
                    if (!pending_commits.emplace(offset_begin, std::move(commit)).second) {
                        error = "duplicate prepared real-time offset range";
                        break;
                    }
                }
            }
            if (!error.empty()) {
                state.RecordError(error);
            }
            state.progress_cv.notify_all();
        };

        std::thread write_thread([&]() {
            state.WaitForStart();
            for (int64_t batch_index = 0; batch_index < kBatchCount && !state.ShouldStop();
                 ++batch_index) {
                std::vector<Row> rows = primary_key
                                            ? pk_batches[static_cast<size_t>(batch_index)]
                                            : MakeRows(batch_index * kRowsPerBatch, kRowsPerBatch,
                                                       /*partition=*/"p0");
                Result<std::unique_ptr<RecordBatch>> batch_result =
                    primary_key ? MakeBatch(rows, /*partitioned=*/false, /*bucket=*/0,
                                            pk_row_kinds[static_cast<size_t>(batch_index)])
                                : MakeBatch(rows, /*partitioned=*/false);
                if (state.RecordErrorIfNotOk(batch_result)) {
                    break;
                }
                Status status = writer->Write(std::move(batch_result).value());
                if (state.RecordErrorIfNotOk(status)) {
                    break;
                }
                std::this_thread::sleep_for(std::chrono::milliseconds(2));
                // Pause midway until one refresh completes to guarantee write and refresh overlap.
                if (batch_index + 1 == kBatchCount / 2) {
                    auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(30);
                    while (refresh_count.load(std::memory_order_acquire) == 0 &&
                           !state.ShouldStop() && std::chrono::steady_clock::now() < deadline) {
                        std::this_thread::sleep_for(std::chrono::milliseconds(1));
                    }
                    if (refresh_count.load(std::memory_order_acquire) == 0 && !state.ShouldStop()) {
                        state.RecordError("timed out waiting for a refresh while writing");
                        break;
                    }
                }
            }
            writer_done.store(true, std::memory_order_release);
        });

        std::vector<std::thread> prepare_threads;
        prepare_threads.reserve(kPrepareThreadCount);
        for (int32_t thread_index = 0; thread_index < kPrepareThreadCount; ++thread_index) {
            prepare_threads.emplace_back([&, thread_index]() {
                state.WaitForStart();
                do {
                    int64_t identifier = next_prepare_identifier.fetch_add(1);
                    Result<std::vector<RealtimeCommitProgress>> result =
                        writer->PrepareCommitWithProgress(identifier);
                    ++prepare_call_counts[thread_index];
                    if (state.RecordErrorIfNotOk(result)) {
                        break;
                    }
                    enqueue_prepared_commits(std::move(result).value());
                    std::this_thread::sleep_for(std::chrono::milliseconds(1));
                } while (!writer_done.load(std::memory_order_acquire) && !state.ShouldStop());
            });
        }

        std::thread commit_thread([&]() {
            state.WaitForStart();
            int64_t next_offset = 0;
            int64_t commit_identifier = 0;
            while (!state.ShouldStop()) {
                std::optional<RealtimeCommitProgress> next_commit;
                {
                    std::unique_lock<std::mutex> lock(state.mutex);
                    state.progress_cv.wait(lock, [&]() {
                        return state.ShouldStop() || pending_commits.count(next_offset) > 0 ||
                               prepare_done.load(std::memory_order_acquire);
                    });
                    if (state.ShouldStop()) {
                        break;
                    }
                    auto iter = pending_commits.find(next_offset);
                    if (iter == pending_commits.end()) {
                        if (prepare_done.load(std::memory_order_acquire)) {
                            if (!pending_commits.empty()) {
                                lock.unlock();
                                state.RecordError("prepared real-time offset ranges contain a gap");
                            }
                            break;
                        }
                        continue;
                    }
                    next_commit = std::move(iter->second);
                    pending_commits.erase(iter);
                }

                std::vector<RealtimeCommitProgress> commits;
                commits.push_back(std::move(next_commit).value());
                int64_t committed_end_offset = commits[0].offset_range.end;
                Result<int64_t> commit_result = Commit(commits, commit_identifier++);
                if (state.RecordErrorIfNotOk(commit_result)) {
                    break;
                }
                next_offset = committed_end_offset;
                {
                    std::lock_guard<std::mutex> lock(state.mutex);
                    pending_snapshot_ids.push_back(std::move(commit_result).value());
                }
                ++commit_count;
                state.snapshot_cv.notify_all();
            }
            commit_done.store(true, std::memory_order_release);
            state.snapshot_cv.notify_all();
        });

        std::thread refresh_thread([&]() {
            state.WaitForStart();
            while (!state.ShouldStop()) {
                std::optional<int64_t> snapshot_id;
                {
                    std::unique_lock<std::mutex> lock(state.mutex);
                    state.snapshot_cv.wait(lock, [&]() {
                        return state.ShouldStop() || !pending_snapshot_ids.empty() ||
                               commit_done.load(std::memory_order_acquire);
                    });
                    if (state.ShouldStop()) {
                        break;
                    }
                    if (pending_snapshot_ids.empty()) {
                        if (commit_done.load(std::memory_order_acquire)) {
                            break;
                        }
                        continue;
                    }
                    snapshot_id = pending_snapshot_ids.front();
                    pending_snapshot_ids.pop_front();
                }
                Status status = writer->RefreshCommittedSnapshot(snapshot_id.value());
                if (state.RecordErrorIfNotOk(status)) {
                    break;
                }
                ++refresh_count;
            }
            refresh_done.store(true, std::memory_order_release);
        });

        std::vector<std::thread> read_threads;
        read_threads.reserve(kReadThreadCount);
        for (int32_t thread_index = 0; thread_index < kReadThreadCount; ++thread_index) {
            read_threads.emplace_back([&, thread_index]() {
                state.WaitForStart();
                while (!refresh_done.load(std::memory_order_acquire) && !state.ShouldStop()) {
                    Result<std::vector<Row>> result = ReadRows(realtime_context);
                    ++read_call_counts[thread_index];
                    if (state.RecordErrorIfNotOk(result)) {
                        break;
                    }
                    Status status = validate_read(result.value());
                    if (state.RecordErrorIfNotOk(status)) {
                        break;
                    }
                    std::this_thread::sleep_for(std::chrono::milliseconds(1));
                }
            });
        }

        constexpr int32_t kWorkerCount = 1 + kPrepareThreadCount + 1 + 1 + kReadThreadCount;
        state.StartWhenReady(kWorkerCount);

        write_thread.join();
        for (std::thread& thread : prepare_threads) {
            thread.join();
        }
        if (!state.ShouldStop()) {
            Result<std::vector<RealtimeCommitProgress>> final_result =
                writer->PrepareCommitWithProgress(next_prepare_identifier.fetch_add(1));
            if (!state.RecordErrorIfNotOk(final_result)) {
                enqueue_prepared_commits(std::move(final_result).value());
            }
        }
        prepare_done.store(true, std::memory_order_release);
        state.progress_cv.notify_all();

        commit_thread.join();
        refresh_thread.join();
        for (std::thread& thread : read_threads) {
            thread.join();
        }

        ASSERT_TRUE(state.Errors().empty())
            << (state.Errors().empty() ? "" : state.Errors().front());
        for (int32_t call_count : prepare_call_counts) {
            ASSERT_GT(call_count, 0);
        }
        for (int32_t call_count : read_call_counts) {
            ASSERT_GT(call_count, 0);
        }
        ASSERT_GE(commit_count.load(), 2);
        ASSERT_GE(refresh_count.load(), 2);
        ASSERT_OK_AND_ASSIGN(std::vector<Row> final_rows, ReadRows(realtime_context));
        if (primary_key) {
            ASSERT_EQ(pk_expected_states.back(), final_rows);
        } else {
            ASSERT_EQ(total_rows, static_cast<int64_t>(final_rows.size()));
            ASSERT_OK(ValidateReadPrefix(final_rows, total_rows));
        }
        ASSERT_OK_AND_ASSIGN(RealtimeOffsetMap committed_offsets, ReadCommittedOffsets());
        ASSERT_EQ(total_rows,
                  committed_offsets.at(RealtimePartitionBucket(/*partition=*/{}, /*bucket=*/0)));
        ASSERT_OK_AND_ASSIGN(uint64_t memory_usage, GetRealtimeMemoryUsage(realtime_context));
        ASSERT_EQ(0, memory_usage);
        ASSERT_OK(writer->Close());
    };
    run_test(/*primary_key=*/false);
    ASSERT_FALSE(HasFatalFailure());
    run_test(/*primary_key=*/true);
    ASSERT_FALSE(HasFatalFailure());
}

TEST_F(RealtimeWriteInteTest, TestMultiplePartitions) {
    CreateTable(/*partition_keys=*/{"pt"});
    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
                         RealtimeContext::Create());
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
                         CreateRealtimeWriter(realtime_context));
    std::vector<std::vector<Row>> disk_rows;
    for (int64_t partition_index = 0; partition_index < 2; ++partition_index) {
        std::string partition = "p" + std::to_string(partition_index);
        std::vector<Row> rows = MakeRows(partition_index * 10, /*count=*/10, partition);
        ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> batch,
                             MakeBatch(rows, /*partitioned=*/true));
        ASSERT_OK(writer->Write(std::move(batch)));
        disk_rows.push_back(std::move(rows));
    }
    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> disk_commits,
                         writer->PrepareCommitWithProgress(/*commit_identifier=*/0));
    ASSERT_EQ(2, disk_commits.size());
    ASSERT_OK(Commit(disk_commits, /*commit_identifier=*/0));

    std::vector<Row> p0_memory_rows = MakeRows(/*first_id=*/20, /*count=*/5, /*partition=*/"p0");
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> p0_memory_batch,
                         MakeBatch(p0_memory_rows, /*partitioned=*/true));
    ASSERT_OK(writer->Write(std::move(p0_memory_batch)));
    std::vector<Row> p2_memory_rows = MakeRows(/*first_id=*/30, /*count=*/5, /*partition=*/"p2");
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> p2_memory_batch,
                         MakeBatch(p2_memory_rows, /*partitioned=*/true));
    ASSERT_OK(writer->Write(std::move(p2_memory_batch)));

    // p0 has disk and memory rows, p1 is disk-only, and p2 is memory-only.
    std::vector<Row> expected_rows = disk_rows[0];
    expected_rows.insert(expected_rows.end(), p0_memory_rows.begin(), p0_memory_rows.end());
    expected_rows.insert(expected_rows.end(), disk_rows[1].begin(), disk_rows[1].end());
    expected_rows.insert(expected_rows.end(), p2_memory_rows.begin(), p2_memory_rows.end());
    ASSERT_OK_AND_ASSIGN(std::vector<Row> actual_rows, ReadRows(realtime_context));
    ASSERT_EQ(expected_rows, actual_rows);

    ASSERT_OK_AND_ASSIGN(RealtimeOffsetMap committed_offsets, ReadCommittedOffsets());
    ASSERT_EQ(2, committed_offsets.size());
    for (int64_t partition_index = 0; partition_index < 2; ++partition_index) {
        RealtimePartitionBucket partition_bucket({{"pt", "p" + std::to_string(partition_index)}},
                                                 /*bucket=*/0);
        ASSERT_EQ(10, committed_offsets.at(partition_bucket));
    }
    ASSERT_EQ(committed_offsets.end(),
              committed_offsets.find(RealtimePartitionBucket({{"pt", "p2"}}, /*bucket=*/0)));

    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> final_commits,
                         writer->PrepareCommitWithProgress(/*commit_identifier=*/1));
    ASSERT_EQ(2, final_commits.size());
    ASSERT_OK_AND_ASSIGN(int64_t final_snapshot_id, Commit(final_commits, /*commit_identifier=*/1));
    ASSERT_OK(writer->RefreshCommittedSnapshot(final_snapshot_id));
    ASSERT_OK_AND_ASSIGN(std::vector<Row> committed_rows, ReadRows(realtime_context));
    ASSERT_EQ(expected_rows, committed_rows);
    ASSERT_OK(writer->Close());
}

TEST_F(RealtimeWriteInteTest, TestDropPartitionRequiresReopenRealtimeContext) {
    CreateTable(/*partition_keys=*/{"pt"});
    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
                         RealtimeContext::Create());
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
                         CreateRealtimeWriter(realtime_context));

    std::vector<Row> disk_rows = MakeRows(/*first_id=*/0, /*count=*/3, /*partition=*/"p0");
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> disk_batch,
                         MakeBatch(disk_rows, /*partitioned=*/true));
    ASSERT_OK(writer->Write(std::move(disk_batch)));
    std::vector<Row> retained_disk_rows =
        MakeRows(/*first_id=*/10, /*count=*/3, /*partition=*/"p1");
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> retained_disk_batch,
                         MakeBatch(retained_disk_rows, /*partitioned=*/true));
    ASSERT_OK(writer->Write(std::move(retained_disk_batch)));
    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> disk_commits,
                         writer->PrepareCommitWithProgress(/*commit_identifier=*/0));
    ASSERT_OK_AND_ASSIGN(int64_t disk_snapshot_id, Commit(disk_commits, /*commit_identifier=*/0));
    ASSERT_OK(writer->RefreshCommittedSnapshot(disk_snapshot_id));

    std::vector<Row> memory_rows = MakeRows(/*first_id=*/3, /*count=*/2, /*partition=*/"p0");
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> memory_batch,
                         MakeBatch(memory_rows, /*partitioned=*/true));
    ASSERT_OK(writer->Write(std::move(memory_batch)));
    std::vector<Row> rows_before_drop = disk_rows;
    rows_before_drop.insert(rows_before_drop.end(), memory_rows.begin(), memory_rows.end());
    rows_before_drop.insert(rows_before_drop.end(), retained_disk_rows.begin(),
                            retained_disk_rows.end());
    ASSERT_OK_AND_ASSIGN(std::vector<Row> actual_rows_before_drop, ReadRows(realtime_context));
    ASSERT_EQ(rows_before_drop, actual_rows_before_drop);

    ASSERT_OK_AND_ASSIGN(int64_t drop_snapshot_id,
                         DropPartition({{"pt", "p0"}}, /*commit_identifier=*/1));
    RealtimePartitionBucket partition_bucket({{"pt", "p0"}}, /*bucket=*/0);
    RealtimePartitionBucket retained_partition_bucket({{"pt", "p1"}}, /*bucket=*/0);
    ASSERT_OK_AND_ASSIGN(RealtimeOffsetMap offsets_after_drop, ReadCommittedOffsets());
    ASSERT_EQ(1, offsets_after_drop.size());
    ASSERT_EQ(3, offsets_after_drop.at(retained_partition_bucket));
    ASSERT_EQ(offsets_after_drop.end(), offsets_after_drop.find(partition_bucket));
    ASSERT_OK_AND_ASSIGN(uint64_t memory_usage_before_refresh,
                         GetRealtimeMemoryUsage(realtime_context));
    ASSERT_NOK_WITH_MSG(writer->RefreshCommittedSnapshot(drop_snapshot_id),
                        "recreate RealtimeContext");
    ASSERT_OK_AND_ASSIGN(uint64_t memory_usage_after_refresh,
                         GetRealtimeMemoryUsage(realtime_context));
    ASSERT_EQ(memory_usage_before_refresh, memory_usage_after_refresh);
    ASSERT_NOK_WITH_MSG(writer->Close(), "not covered by a successful");
    writer.reset();
    realtime_context.reset();

    // Reopen with p1's retained progress and replay p0 input that existed only in the old context.
    ASSERT_OK_AND_ASSIGN(realtime_context, RealtimeContext::Create());
    ASSERT_OK_AND_ASSIGN(writer, CreateRealtimeWriter(realtime_context));
    ResetExternalOffset({{"pt", "p0"}}, /*bucket=*/0, /*next_offset=*/0);
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> replay_batch,
                         MakeBatch(memory_rows, /*partitioned=*/true));
    ASSERT_OK(writer->Write(std::move(replay_batch)));
    ASSERT_OK_AND_ASSIGN(std::vector<Row> replayed_rows, ReadRows(realtime_context));
    // The retained p1 disk split is read before the tail real-time split containing p0.
    std::vector<Row> expected_replayed_rows = retained_disk_rows;
    expected_replayed_rows.insert(expected_replayed_rows.end(), memory_rows.begin(),
                                  memory_rows.end());
    ASSERT_EQ(expected_replayed_rows, replayed_rows);

    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> memory_commits,
                         writer->PrepareCommitWithProgress(/*commit_identifier=*/2));
    ASSERT_EQ(1, memory_commits.size());
    ASSERT_EQ(OffsetRange(0, 2), memory_commits[0].offset_range);
    ASSERT_OK_AND_ASSIGN(int64_t memory_snapshot_id,
                         Commit(memory_commits, /*commit_identifier=*/2));
    ASSERT_OK(writer->RefreshCommittedSnapshot(memory_snapshot_id));
    ASSERT_OK_AND_ASSIGN(std::vector<Row> rows_after_memory_commit, ReadRows(realtime_context));
    std::vector<Row> expected_committed_rows = memory_rows;
    expected_committed_rows.insert(expected_committed_rows.end(), retained_disk_rows.begin(),
                                   retained_disk_rows.end());
    ASSERT_EQ(expected_committed_rows, rows_after_memory_commit);
    ASSERT_OK_AND_ASSIGN(uint64_t final_memory_usage, GetRealtimeMemoryUsage(realtime_context));
    ASSERT_EQ(0, final_memory_usage);
    ASSERT_OK_AND_ASSIGN(RealtimeOffsetMap final_offsets, ReadCommittedOffsets());
    ASSERT_EQ(2, final_offsets.size());
    ASSERT_EQ(2, final_offsets.at(partition_bucket));
    ASSERT_EQ(3, final_offsets.at(retained_partition_bucket));
    ASSERT_OK(writer->Close());
}

TEST_F(RealtimeWriteInteTest, TestDropInactivePartitionDoesNotRequireReopenRealtimeContext) {
    CreateTable(/*partition_keys=*/{"pt"});

    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> seed_context, RealtimeContext::Create());
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> seed_writer,
                         CreateRealtimeWriter(seed_context));
    for (int64_t partition_index = 0; partition_index < 2; ++partition_index) {
        std::string partition = "p" + std::to_string(partition_index);
        ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> batch,
                             MakeBatch(MakeRows(partition_index * 10, /*count=*/3, partition),
                                       /*partitioned=*/true));
        ASSERT_OK(seed_writer->Write(std::move(batch)));
    }
    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> seed_commits,
                         seed_writer->PrepareCommitWithProgress(/*commit_identifier=*/0));
    ASSERT_EQ(2, seed_commits.size());
    ASSERT_OK(Commit(seed_commits, /*commit_identifier=*/0));
    ASSERT_OK(seed_writer->Close());
    seed_writer.reset();
    seed_context.reset();

    // The new context loads offsets for both partitions, but creates a store only for p0.
    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
                         RealtimeContext::Create());
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer,
                         CreateRealtimeWriter(realtime_context));
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> p0_batch,
                         MakeBatch(MakeRows(/*first_id=*/20, /*count=*/1, /*partition=*/"p0"),
                                   /*partitioned=*/true));
    ASSERT_OK(writer->Write(std::move(p0_batch)));

    ASSERT_OK_AND_ASSIGN(int64_t drop_snapshot_id,
                         DropPartition({{"pt", "p1"}}, /*commit_identifier=*/1));
    ASSERT_OK(writer->RefreshCommittedSnapshot(drop_snapshot_id));
    const RealtimePartitionBucket p0_partition_bucket({{"pt", "p0"}}, /*bucket=*/0);
    const RealtimePartitionBucket p1_partition_bucket({{"pt", "p1"}}, /*bucket=*/0);
    ASSERT_OK_AND_ASSIGN(RealtimeOffsetMap offsets_after_drop, ReadCommittedOffsets());
    ASSERT_EQ(1, offsets_after_drop.size());
    ASSERT_EQ(3, offsets_after_drop.at(p0_partition_bucket));
    ASSERT_EQ(offsets_after_drop.end(), offsets_after_drop.find(p1_partition_bucket));

    // Since p1 was never active in this context, writing it after the drop starts from zero.
    ResetExternalOffset({{"pt", "p1"}}, /*bucket=*/0, /*next_offset=*/0);
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> p1_batch,
                         MakeBatch(MakeRows(/*first_id=*/30, /*count=*/2, /*partition=*/"p1"),
                                   /*partitioned=*/true));
    ASSERT_OK(writer->Write(std::move(p1_batch)));
    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> commits,
                         writer->PrepareCommitWithProgress(/*commit_identifier=*/2));
    ASSERT_EQ(2, commits.size());
    auto p1_commit =
        std::find_if(commits.begin(), commits.end(), [&](const RealtimeCommitProgress& commit) {
            return commit.partition_bucket == p1_partition_bucket;
        });
    ASSERT_NE(commits.end(), p1_commit);
    ASSERT_EQ(OffsetRange(0, 2), p1_commit->offset_range);
    ASSERT_OK(writer->Close());
}

TEST_F(RealtimeWriteInteTest, TestDropDatePartitionRemovesOffsetWithLegacyPartitionName) {
    CheckDropDatePartitionRemovesOffset(/*legacy_partition_name_enabled=*/true);
}

TEST_F(RealtimeWriteInteTest, TestDropDatePartitionRemovesOffsetWithoutLegacyPartitionName) {
    CheckDropDatePartitionRemovesOffset(/*legacy_partition_name_enabled=*/false);
}

TEST_F(RealtimeWriteInteTest, TestMultipleBucketsRestoreIndependentOffsets) {
    options_[Options::BUCKET] = "2";
    CreateTable(/*partition_keys=*/{});

    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> first_writer, CreateRealtimeWriter());
    std::vector<Row> bucket0_disk_rows = MakeRows(/*first_id=*/0, /*count=*/2, /*partition=*/"p0");
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> bucket0_disk_batch,
                         MakeBatch(bucket0_disk_rows, /*partitioned=*/false, /*bucket=*/0));
    ASSERT_OK(first_writer->Write(std::move(bucket0_disk_batch)));
    std::vector<Row> bucket1_disk_rows = MakeRows(/*first_id=*/10, /*count=*/3, /*partition=*/"p0");
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> bucket1_disk_batch,
                         MakeBatch(bucket1_disk_rows, /*partitioned=*/false, /*bucket=*/1));
    ASSERT_OK(first_writer->Write(std::move(bucket1_disk_batch)));
    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> first_commits,
                         first_writer->PrepareCommitWithProgress(/*commit_identifier=*/0));
    ASSERT_EQ(2, first_commits.size());
    ASSERT_OK(Commit(first_commits, /*commit_identifier=*/0));
    ASSERT_OK(first_writer->Close());

    ASSERT_OK_AND_ASSIGN(std::shared_ptr<RealtimeContext> realtime_context,
                         RealtimeContext::Create());
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> second_writer,
                         CreateRealtimeWriter(realtime_context));
    std::vector<Row> bucket0_memory_rows =
        MakeRows(/*first_id=*/2, /*count=*/1, /*partition=*/"p0");
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> bucket0_memory_batch,
                         MakeBatch(bucket0_memory_rows, /*partitioned=*/false, /*bucket=*/0));
    ASSERT_OK(second_writer->Write(std::move(bucket0_memory_batch)));
    std::vector<Row> bucket1_memory_rows =
        MakeRows(/*first_id=*/13, /*count=*/1, /*partition=*/"p0");
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> bucket1_memory_batch,
                         MakeBatch(bucket1_memory_rows, /*partitioned=*/false, /*bucket=*/1));
    ASSERT_OK(second_writer->Write(std::move(bucket1_memory_batch)));
    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> second_commits,
                         second_writer->PrepareCommitWithProgress(/*commit_identifier=*/1));
    ASSERT_EQ(2, second_commits.size());
    std::map<int32_t, OffsetRange> prepared_ranges;
    for (const RealtimeCommitProgress& commit : second_commits) {
        prepared_ranges.emplace(commit.partition_bucket.bucket, commit.offset_range);
    }
    ASSERT_EQ(OffsetRange(2, 3), prepared_ranges.at(0));
    ASSERT_EQ(OffsetRange(3, 4), prepared_ranges.at(1));

    ASSERT_OK_AND_ASSIGN(std::vector<Row> actual_rows, ReadRows(realtime_context));
    std::vector<Row> expected_rows = bucket0_disk_rows;
    expected_rows.insert(expected_rows.end(), bucket0_memory_rows.begin(),
                         bucket0_memory_rows.end());
    expected_rows.insert(expected_rows.end(), bucket1_disk_rows.begin(), bucket1_disk_rows.end());
    expected_rows.insert(expected_rows.end(), bucket1_memory_rows.begin(),
                         bucket1_memory_rows.end());
    std::sort(expected_rows.begin(), expected_rows.end());
    std::sort(actual_rows.begin(), actual_rows.end());
    ASSERT_EQ(expected_rows, actual_rows);

    ASSERT_OK(Commit(second_commits, /*commit_identifier=*/1));
    ASSERT_OK(second_writer->Close());
    ASSERT_OK_AND_ASSIGN(RealtimeOffsetMap committed_offsets, ReadCommittedOffsets());
    ASSERT_EQ(2, committed_offsets.size());
    ASSERT_EQ(3, committed_offsets.at(RealtimePartitionBucket(/*partition=*/{}, /*bucket=*/0)));
    ASSERT_EQ(4, committed_offsets.at(RealtimePartitionBucket(/*partition=*/{}, /*bucket=*/1)));
}

TEST_F(RealtimeWriteInteTest, TestRestoreOffsetFromCommittedSnapshot) {
    CreateTable(/*partition_keys=*/{});

    std::vector<Row> first_rows = MakeRows(/*first_id=*/0, /*count=*/3, /*partition=*/"p0");
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> first_writer, CreateRealtimeWriter());
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> first_batch,
                         MakeBatch(first_rows, /*partitioned=*/false));
    ASSERT_OK(first_writer->Write(std::move(first_batch)));
    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> first_commits,
                         first_writer->PrepareCommitWithProgress(/*commit_identifier=*/0));
    ASSERT_EQ(1, first_commits.size());
    ASSERT_EQ(OffsetRange(0, 3), first_commits[0].offset_range);
    ASSERT_OK(Commit(first_commits, /*commit_identifier=*/0));
    ASSERT_OK(first_writer->Close());

    std::vector<Row> second_rows = MakeRows(/*first_id=*/3, /*count=*/2, /*partition=*/"p0");
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> second_writer, CreateRealtimeWriter());
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> second_batch,
                         MakeBatch(second_rows, /*partitioned=*/false));
    ASSERT_OK(second_writer->Write(std::move(second_batch)));
    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> second_commits,
                         second_writer->PrepareCommitWithProgress(/*commit_identifier=*/1));
    ASSERT_EQ(1, second_commits.size());
    ASSERT_EQ(OffsetRange(3, 5), second_commits[0].offset_range);

    std::vector<Row> expected_rows = MakeRows(/*first_id=*/0, /*count=*/5, /*partition=*/"p0");
    FinalizeCommitAndCheck(second_writer.get(), std::move(second_commits),
                           /*prepare_identifier=*/1, std::move(expected_rows));

    RealtimePartitionBucket partition_bucket(/*partition=*/{}, /*bucket=*/0);
    ASSERT_OK_AND_ASSIGN(RealtimeOffsetMap second_committed_offsets, ReadCommittedOffsets());
    ASSERT_EQ(5, second_committed_offsets.at(partition_bucket));
}

TEST_F(RealtimeWriteInteTest, TestReadRealtimeOffsetsFromExactSnapshot) {
    options_[Options::BUCKET] = "2";
    CreateTable(/*partition_keys=*/{"pt"});

    ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreWrite> writer, CreateRealtimeWriter());
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> p0_bucket0_batch,
                         MakeBatch(MakeRows(/*first_id=*/0, /*count=*/2, /*partition=*/"p0"),
                                   /*partitioned=*/true, /*bucket=*/0));
    ASSERT_OK(writer->Write(std::move(p0_bucket0_batch)));
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> p0_bucket1_batch,
                         MakeBatch(MakeRows(/*first_id=*/10, /*count=*/3, /*partition=*/"p0"),
                                   /*partitioned=*/true, /*bucket=*/1));
    ASSERT_OK(writer->Write(std::move(p0_bucket1_batch)));
    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> p1_bucket0_batch,
                         MakeBatch(MakeRows(/*first_id=*/20, /*count=*/1, /*partition=*/"p1"),
                                   /*partitioned=*/true, /*bucket=*/0));
    ASSERT_OK(writer->Write(std::move(p1_bucket0_batch)));
    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> first_commits,
                         writer->PrepareCommitWithProgress(/*commit_identifier=*/0));
    ASSERT_OK_AND_ASSIGN(int64_t first_snapshot_id, Commit(first_commits, /*commit_identifier=*/0));
    ASSERT_OK(writer->RefreshCommittedSnapshot(first_snapshot_id));

    const RealtimePartitionBucket p0_bucket0({{"pt", "p0"}}, /*bucket=*/0);
    const RealtimePartitionBucket p0_bucket1({{"pt", "p0"}}, /*bucket=*/1);
    const RealtimePartitionBucket p1_bucket0({{"pt", "p1"}}, /*bucket=*/0);
    ASSERT_OK_AND_ASSIGN(
        RealtimeOffsetMap first_offsets,
        RealtimeSnapshotOffsets::ReadAllOffsets(table_path_, /*branch=*/"", first_snapshot_id,
                                                options_, /*file_system=*/nullptr));
    ASSERT_EQ(3, first_offsets.size());
    ASSERT_EQ(2, first_offsets.at(p0_bucket0));
    ASSERT_EQ(3, first_offsets.at(p0_bucket1));
    ASSERT_EQ(1, first_offsets.at(p1_bucket0));
    ASSERT_OK_AND_ASSIGN(int64_t p0_bucket1_offset,
                         RealtimeSnapshotOffsets::ReadOffset(
                             table_path_, /*branch=*/"", first_snapshot_id, p0_bucket1, options_,
                             /*file_system=*/nullptr));
    ASSERT_EQ(3, p0_bucket1_offset);
    ASSERT_OK_AND_ASSIGN(int64_t missing_offset,
                         RealtimeSnapshotOffsets::ReadOffset(
                             table_path_, /*branch=*/"", first_snapshot_id,
                             RealtimePartitionBucket({{"pt", "missing"}}, /*bucket=*/0), options_,
                             /*file_system=*/nullptr));
    ASSERT_EQ(-1, missing_offset);

    ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> second_batch,
                         MakeBatch(MakeRows(/*first_id=*/2, /*count=*/1, /*partition=*/"p0"),
                                   /*partitioned=*/true, /*bucket=*/0));
    ASSERT_OK(writer->Write(std::move(second_batch)));
    ASSERT_OK_AND_ASSIGN(std::vector<RealtimeCommitProgress> second_commits,
                         writer->PrepareCommitWithProgress(/*commit_identifier=*/1));
    ASSERT_OK_AND_ASSIGN(int64_t second_snapshot_id,
                         Commit(second_commits, /*commit_identifier=*/1));
    ASSERT_OK(writer->RefreshCommittedSnapshot(second_snapshot_id));

    ASSERT_OK_AND_ASSIGN(int64_t first_snapshot_offset,
                         RealtimeSnapshotOffsets::ReadOffset(
                             table_path_, /*branch=*/"", first_snapshot_id, p0_bucket0, options_,
                             /*file_system=*/nullptr));
    ASSERT_OK_AND_ASSIGN(int64_t second_snapshot_offset,
                         RealtimeSnapshotOffsets::ReadOffset(
                             table_path_, /*branch=*/"", second_snapshot_id, p0_bucket0, options_,
                             /*file_system=*/nullptr));
    ASSERT_EQ(2, first_snapshot_offset);
    ASSERT_EQ(3, second_snapshot_offset);
    ASSERT_OK(writer->Close());
}

}  // namespace paimon::test