blob: 0f6c9ef95e1dcf84df6e83f9ccc483ecf401feed [file]
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
#include "paimon/io/buffered_input_stream.h"
#include <utility>
#include "gtest/gtest.h"
#include "paimon/common/io/memory_segment_output_stream.h"
#include "paimon/common/memory/memory_segment_utils.h"
#include "paimon/io/byte_array_input_stream.h"
#include "paimon/memory/bytes.h"
#include "paimon/memory/memory_pool.h"
#include "paimon/testing/utils/testharness.h"
namespace paimon::test {
TEST(BufferedInputStreamTest, TestSimple) {
auto pool = GetDefaultPool();
auto output_stream = std::make_unique<MemorySegmentOutputStream>(/*segment_size=*/8, pool);
std::string str = "abcdef";
auto bytes = std::make_shared<Bytes>(str, pool.get());
output_stream->WriteBytes(bytes);
auto out_bytes = MemorySegmentUtils::CopyToBytes(output_stream->Segments(), 0,
output_stream->CurrentSize(), pool.get());
auto in = std::make_shared<ByteArrayInputStream>(out_bytes->data(), out_bytes->size());
auto input_stream = std::make_shared<BufferedInputStream>(in, /*buffer_size=*/4, pool.get());
ASSERT_EQ(6, input_stream->Length().value());
ASSERT_TRUE(input_stream->GetUri().value().empty());
// read from pos 0
std::string value(3, '\0');
ASSERT_EQ(3, input_stream->Read(value.data(), value.size()).value());
ASSERT_EQ("abc", value);
ASSERT_EQ(3, input_stream->GetPos().value());
ASSERT_EQ(3, input_stream->Read(value.data(), value.size()).value());
ASSERT_EQ("def", value);
ASSERT_EQ(6, input_stream->GetPos().value());
// read from pos 1
ASSERT_NOK_WITH_MSG(input_stream->Read(value.data(), value.size(), /*offset=*/1),
"BufferedInputStream does not support Read from offset");
ASSERT_EQ(6, input_stream->GetPos().value());
// seek to pos 3
ASSERT_OK(input_stream->Seek(3, SeekOrigin::FS_SEEK_SET));
ASSERT_EQ(3, input_stream->Read(value.data(), value.size()).value());
ASSERT_EQ("def", value);
ASSERT_EQ(6, input_stream->GetPos().value());
// seek to pos 2
ASSERT_OK(input_stream->Seek(-4, SeekOrigin::FS_SEEK_END));
ASSERT_EQ(3, input_stream->Read(value.data(), value.size()).value());
ASSERT_EQ("cde", value);
ASSERT_EQ(5, input_stream->GetPos().value());
// seek to pos 1
ASSERT_OK(input_stream->Seek(-4, SeekOrigin::FS_SEEK_CUR));
ASSERT_EQ(3, input_stream->Read(value.data(), value.size()).value());
ASSERT_EQ("bcd", value);
ASSERT_EQ(4, input_stream->GetPos().value());
// test exceed eof, seek to pos 4, want to read 4 bytes
ASSERT_OK(input_stream->Seek(-2, SeekOrigin::FS_SEEK_END));
ASSERT_NOK_WITH_MSG(input_stream->Read(value.data(), value.size()),
"InnerRead failed, after Fill(), still no bytes available (may read eof), "
"but expect read 1 bytes");
// test invalid seek
ASSERT_NOK_WITH_MSG(input_stream->Seek(100, SeekOrigin::FS_SEEK_CUR),
"invalid seek, after seek, current pos 106, length 6");
// test ReadAsync not implemented
bool read_finished = false;
auto callback = [&](Status status) {
ASSERT_TRUE(status.IsNotImplemented());
read_finished = true;
};
input_stream->ReadAsync(value.data(), value.size(), /*offset=*/0, callback);
ASSERT_TRUE(read_finished);
ASSERT_OK(input_stream->Close());
}
TEST(BufferedInputStreamTest, TestSeek) {
// Data: "0123456789abcdef" (16 bytes), buffer_size = 8
auto pool = GetDefaultPool();
std::string data = "0123456789abcdef";
auto in = std::make_shared<ByteArrayInputStream>(reinterpret_cast<const char*>(data.data()),
data.size());
auto stream = std::make_shared<BufferedInputStream>(in, /*buffer_size=*/8, pool.get());
// Helper: verify buffer_ content matches expected substring of data
auto check_buffer = [&](const BufferedInputStream& s, const std::string& expected_content,
int32_t expected_pos, int32_t expected_count) {
ASSERT_EQ(s.pos_, expected_pos);
ASSERT_EQ(s.count_, expected_count);
std::string actual(s.buffer_->data(), s.buffer_->data() + expected_count);
ASSERT_EQ(actual, expected_content) << "buffer content mismatch: actual=\"" << actual
<< "\", expected=\"" << expected_content << "\"";
};
std::string buf(4, '\0');
// Initial state: buffer empty
ASSERT_EQ(stream->pos_, 0);
ASSERT_EQ(stream->count_, 0);
// FS_SEEK_SET slow path (buffer empty, count_==0): seek to pos 4, read "4567"
// First Read calls Fill: reads 8 bytes from pos 4 -> buffer = "456789ab", count_=8.
// Then consumes 4 bytes -> pos_=4.
{
ASSERT_OK(stream->Seek(4, SeekOrigin::FS_SEEK_SET));
// After seek: slow path, buffer invalidated
ASSERT_EQ(stream->pos_, 0);
ASSERT_EQ(stream->count_, 0);
ASSERT_EQ(4, stream->GetPos().value());
ASSERT_EQ(4, stream->Read(buf.data(), 4).value());
ASSERT_EQ("4567", buf);
check_buffer(*stream, "456789ab", 4, 8);
}
// FS_SEEK_CUR buffer hit: pos=8, seek -4 -> target=4, inside buffer [4..12)
// Fast path: only adjusts pos_ to 0, count_ stays 8, buffer unchanged.
{
ASSERT_OK(stream->Seek(-4, SeekOrigin::FS_SEEK_CUR));
check_buffer(*stream, "456789ab", 0, 8);
ASSERT_EQ(4, stream->GetPos().value());
ASSERT_EQ(4, stream->Read(buf.data(), 4).value());
ASSERT_EQ("4567", buf);
check_buffer(*stream, "456789ab", 4, 8);
}
// FS_SEEK_SET buffer hit: pos=8, seek to 5, inside buffer [4..12)
// Fast path: pos_ = 5 - 4 = 1, count_ stays 8, buffer unchanged.
{
ASSERT_OK(stream->Seek(5, SeekOrigin::FS_SEEK_SET));
check_buffer(*stream, "456789ab", 1, 8);
ASSERT_EQ(5, stream->GetPos().value());
ASSERT_EQ(4, stream->Read(buf.data(), 4).value());
ASSERT_EQ("5678", buf);
check_buffer(*stream, "456789ab", 5, 8);
}
// FS_SEEK_END buffer miss: target = 16 + (-6) = 10
// Current buffer covers [4..12). pos 10 is inside [4..12) -> actually buffer hit!
// Fast path: pos_ = 10 - 4 = 6, count_ stays 8.
{
ASSERT_OK(stream->Seek(-6, SeekOrigin::FS_SEEK_END));
check_buffer(*stream, "456789ab", 6, 8);
ASSERT_EQ(10, stream->GetPos().value());
// Read 4 bytes from pos 10: 2 left in buffer ("ab"), then refill.
ASSERT_EQ(4, stream->Read(buf.data(), 4).value());
ASSERT_EQ("abcd", buf);
// After consuming "ab" (pos_==count_==8), InnerRead calls Fill from in_ pos 12,
// reads [12..16) -> buffer = "cdef", count_=4, then consumes 2 -> pos_=2.
check_buffer(*stream, "cdef", 2, 4);
}
// Buffer miss slow path: seek to pos 0, current buffer covers [12..16).
// Target 0 is outside [12..16) -> slow path, buffer invalidated.
{
ASSERT_OK(stream->Seek(0, SeekOrigin::FS_SEEK_SET));
ASSERT_EQ(stream->pos_, 0);
ASSERT_EQ(stream->count_, 0);
ASSERT_EQ(0, stream->GetPos().value());
ASSERT_EQ(4, stream->Read(buf.data(), 4).value());
ASSERT_EQ("0123", buf);
// Fill reads [0..8) -> buffer = "01234567", count_=8, pos_=4
check_buffer(*stream, "01234567", 4, 8);
}
// Buffer hit at exact boundary start: seek to pos 0 (= buf_start_abs=0)
// Fast path: pos_ = 0, count_ stays 8, buffer unchanged.
{
ASSERT_OK(stream->Seek(0, SeekOrigin::FS_SEEK_SET));
check_buffer(*stream, "01234567", 0, 8);
ASSERT_EQ(4, stream->Read(buf.data(), 4).value());
ASSERT_EQ("0123", buf);
check_buffer(*stream, "01234567", 4, 8);
}
// Buffer hit at exact boundary end: seek to pos 8 (= buf_end_abs=8)
// Fast path: pos_ = 8 == count_, buffer unchanged but next Read triggers refill.
{
ASSERT_OK(stream->Seek(8, SeekOrigin::FS_SEEK_SET));
check_buffer(*stream, "01234567", 8, 8);
ASSERT_EQ(8, stream->GetPos().value());
ASSERT_EQ(4, stream->Read(buf.data(), 4).value());
ASSERT_EQ("89ab", buf);
// pos_==count_ triggered Fill: buffer = "89abcdef", count_=8, pos_=4
check_buffer(*stream, "89abcdef", 4, 8);
}
// FS_SEEK_CUR buffer miss: pos=12, seek -12 -> target=0, outside [8..16)
{
ASSERT_OK(stream->Seek(-12, SeekOrigin::FS_SEEK_CUR));
ASSERT_EQ(stream->pos_, 0);
ASSERT_EQ(stream->count_, 0);
ASSERT_EQ(4, stream->Read(buf.data(), 4).value());
ASSERT_EQ("0123", buf);
check_buffer(*stream, "01234567", 4, 8);
}
// FS_SEEK_END buffer hit: fill buffer near file end, then seek within via FS_SEEK_END.
{
ASSERT_OK(stream->Seek(12, SeekOrigin::FS_SEEK_SET));
// pos 12 is outside current buffer [0..8) -> slow path
ASSERT_EQ(stream->pos_, 0);
ASSERT_EQ(stream->count_, 0);
ASSERT_EQ(4, stream->Read(buf.data(), 4).value());
ASSERT_EQ("cdef", buf);
// Fill from 12: buffer = "cdef", count_=4, pos_=4
check_buffer(*stream, "cdef", 4, 4);
// Seek -4 from end -> target = 12, buffer covers [12..16), 12 is inside.
// Fast path: pos_ = 12 - 12 = 0
ASSERT_OK(stream->Seek(-4, SeekOrigin::FS_SEEK_END));
check_buffer(*stream, "cdef", 0, 4);
ASSERT_EQ(12, stream->GetPos().value());
ASSERT_EQ(4, stream->Read(buf.data(), 4).value());
ASSERT_EQ("cdef", buf);
check_buffer(*stream, "cdef", 4, 4);
}
// Empty buffer (count_==0): fresh stream, seek always takes slow path.
{
auto in2 = std::make_shared<ByteArrayInputStream>(
reinterpret_cast<const char*>(data.data()), data.size());
auto fresh = std::make_shared<BufferedInputStream>(in2, /*buffer_size=*/8, pool.get());
ASSERT_EQ(fresh->pos_, 0);
ASSERT_EQ(fresh->count_, 0);
ASSERT_OK(fresh->Seek(10, SeekOrigin::FS_SEEK_SET));
ASSERT_EQ(fresh->pos_, 0);
ASSERT_EQ(fresh->count_, 0);
ASSERT_EQ(10, fresh->GetPos().value());
ASSERT_EQ(4, fresh->Read(buf.data(), 4).value());
ASSERT_EQ("abcd", buf);
// Fill from 10: buffer = "abcdef", count_=6, pos_=4
check_buffer(*fresh, "abcdef", 4, 6);
}
}
} // namespace paimon::test