blob: 88d79fc09140d75a2f24da7b780c40f7614aea31 [file]
/**
*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include <filesystem>
#include <fstream>
#include "LmdbContentRepository.h"
#include "ResourceClaim.h"
#include "properties/Configure.h"
#include "unit/Catch.h"
#include "unit/ContentRepositoryDependentTests.h"
#include "unit/TestBase.h"
namespace org::apache::nifi::minifi::test {
class LmdbContentRepositoryTests : TestController {
public:
LmdbContentRepositoryTests() {
auto configuration = std::make_shared<org::apache::nifi::minifi::ConfigureImpl>();
configuration->set(minifi::Configure::nifi_dbcontent_repository_directory_default, createTempDirectory().string());
REQUIRE(content_repo_->initialize(configuration));
}
protected:
static constexpr std::string_view test_content_ = "well hello there";
std::shared_ptr<core::repository::LmdbContentRepository> content_repo_ = std::make_shared<core::repository::LmdbContentRepository>();
void writeContent(const minifi::ResourceClaim& claim) {
auto stream = content_repo_->write(claim, false);
stream->write(as_bytes(std::span(test_content_)));
stream->close();
}
};
TEST_CASE("Invalid or empty dbsize configuration value is set", "[lmdb]") {
TestController controller;
LogTestController::getInstance().setDebug<core::repository::LmdbContentRepository>();
auto db_path = controller.createTempDirectory().string();
auto configuration = std::make_shared<org::apache::nifi::minifi::ConfigureImpl>();
auto content_repo = std::make_shared<core::repository::LmdbContentRepository>();
configuration->set(minifi::Configure::nifi_dbcontent_repository_directory_default, db_path);
SECTION("Invalid value") {
configuration->set(minifi::Configure::nifi_content_repository_lmdb_max_db_size, "invalid");
REQUIRE_FALSE(content_repo->initialize(configuration));
REQUIRE(LogTestController::getInstance().contains("Invalid max DB size configuration for LMDB Content Repository: invalid"));
}
SECTION("Empty value") {
configuration->set(minifi::Configure::nifi_content_repository_lmdb_max_db_size, "");
REQUIRE(content_repo->initialize(configuration));
REQUIRE(LogTestController::getInstance().contains("Setting LMDB max DB size to 10737418240 bytes"));
}
}
TEST_CASE("Valid dbsize configuration value is set", "[lmdb]") {
TestController controller;
LogTestController::getInstance().setDebug<core::repository::LmdbContentRepository>();
auto configuration = std::make_shared<org::apache::nifi::minifi::ConfigureImpl>();
auto content_repo = std::make_shared<core::repository::LmdbContentRepository>();
configuration->set(minifi::Configure::nifi_dbcontent_repository_directory_default, controller.createTempDirectory().string());
configuration->set(minifi::Configure::nifi_content_repository_lmdb_max_db_size, "100 MB");
REQUIRE(content_repo->initialize(configuration));
REQUIRE(LogTestController::getInstance().contains("Setting LMDB max DB size to 104857600 bytes"));
}
TEST_CASE("Initialize succeeds when target directory already exists", "[lmdb]") {
TestController controller;
auto db_path = controller.createTempDirectory();
REQUIRE(std::filesystem::exists(db_path));
auto configuration = std::make_shared<org::apache::nifi::minifi::ConfigureImpl>();
auto content_repo = std::make_shared<core::repository::LmdbContentRepository>();
configuration->set(minifi::Configure::nifi_dbcontent_repository_directory_default, db_path.string());
REQUIRE(content_repo->initialize(configuration));
}
TEST_CASE("Initialize fails and is safe to destroy when the directory path is a regular file", "[lmdb]") {
TestController controller;
const auto file_path = controller.createTempDirectory() / "not_a_directory";
{
std::ofstream file(file_path);
file << "this is a regular file, not a directory";
}
REQUIRE(std::filesystem::is_regular_file(file_path));
auto configuration = std::make_shared<org::apache::nifi::minifi::ConfigureImpl>();
configuration->set(minifi::Configure::nifi_dbcontent_repository_directory_default, file_path.string());
auto content_repo = std::make_shared<core::repository::LmdbContentRepository>();
REQUIRE_FALSE(content_repo->initialize(configuration));
}
TEST_CASE_METHOD(LmdbContentRepositoryTests, "Key does not exist in empty database", "[lmdb]") {
auto claim = std::make_shared<minifi::ResourceClaimImpl>(content_repo_);
REQUIRE_FALSE(content_repo_->exists(*claim));
}
TEST_CASE_METHOD(LmdbContentRepositoryTests, "Written value exists", "[lmdb]") {
auto claim = std::make_shared<minifi::ResourceClaimImpl>(content_repo_);
writeContent(*claim);
REQUIRE(content_repo_->exists(*claim));
}
TEST_CASE_METHOD(LmdbContentRepositoryTests, "exists returns false for unrelated claim when other content was written", "[lmdb]") {
auto written_claim = std::make_shared<minifi::ResourceClaimImpl>(content_repo_);
writeContent(*written_claim);
auto unrelated_claim = std::make_shared<minifi::ResourceClaimImpl>(content_repo_);
REQUIRE(content_repo_->exists(*written_claim));
REQUIRE_FALSE(content_repo_->exists(*unrelated_claim));
}
TEST_CASE_METHOD(LmdbContentRepositoryTests, "Empty content claim does not exist", "[lmdb]") {
auto claim = std::make_shared<minifi::ResourceClaimImpl>(content_repo_);
auto write_stream = content_repo_->write(*claim, false);
write_stream->close();
REQUIRE_FALSE(content_repo_->exists(*claim));
}
TEST_CASE_METHOD(LmdbContentRepositoryTests, "Multiple claims coexist independently", "[lmdb]") {
auto claim1 = std::make_shared<minifi::ResourceClaimImpl>(content_repo_);
auto claim2 = std::make_shared<minifi::ResourceClaimImpl>(content_repo_);
auto claim3 = std::make_shared<minifi::ResourceClaimImpl>(content_repo_);
writeContent(*claim1);
writeContent(*claim2);
writeContent(*claim3);
REQUIRE(content_repo_->exists(*claim1));
REQUIRE(content_repo_->exists(*claim2));
REQUIRE(content_repo_->exists(*claim3));
REQUIRE(content_repo_->getRepositoryEntryCount() == 3);
REQUIRE(content_repo_->remove(*claim2));
REQUIRE(content_repo_->exists(*claim1));
REQUIRE_FALSE(content_repo_->exists(*claim2));
REQUIRE(content_repo_->exists(*claim3));
REQUIRE(content_repo_->getRepositoryEntryCount() == 2);
}
TEST_CASE_METHOD(LmdbContentRepositoryTests, "Reading nonexistent claim returns STREAM_ERROR", "[lmdb]") {
auto claim = std::make_shared<minifi::ResourceClaimImpl>(content_repo_);
auto read_stream = content_repo_->read(*claim);
std::vector<std::byte> buffer(8);
REQUIRE(minifi::io::isError(read_stream->read(as_writable_bytes(std::span(buffer)))));
}
TEST_CASE_METHOD(LmdbContentRepositoryTests, "Read written value", "[lmdb]") {
auto claim = std::make_shared<minifi::ResourceClaimImpl>(content_repo_);
writeContent(*claim);
auto read_stream = content_repo_->read(*claim);
std::vector<std::byte> buffer(test_content_.size());
auto bytes_read = read_stream->read(as_writable_bytes(std::span(buffer)));
read_stream->close();
REQUIRE(bytes_read == test_content_.size());
REQUIRE(std::string_view(reinterpret_cast<const char*>(buffer.data()), buffer.size()) == test_content_);
}
TEST_CASE_METHOD(LmdbContentRepositoryTests, "Removing a nonexistent claim succeeds", "[lmdb]") {
auto claim = std::make_shared<minifi::ResourceClaimImpl>(content_repo_);
REQUIRE_FALSE(content_repo_->exists(*claim));
REQUIRE(content_repo_->remove(*claim));
}
TEST_CASE_METHOD(LmdbContentRepositoryTests, "Removing an existing value", "[lmdb]") {
auto claim = std::make_shared<minifi::ResourceClaimImpl>(content_repo_);
writeContent(*claim);
REQUIRE(content_repo_->exists(*claim));
REQUIRE(content_repo_->remove(*claim));
REQUIRE_FALSE(content_repo_->exists(*claim));
}
TEST_CASE_METHOD(LmdbContentRepositoryTests, "clearOrphans is a no-op on an empty repository", "[lmdb]") {
REQUIRE(content_repo_->getRepositoryEntryCount() == 0);
content_repo_->clearOrphans();
REQUIRE(content_repo_->getRepositoryEntryCount() == 0);
}
TEST_CASE_METHOD(LmdbContentRepositoryTests, "Clear orphan values", "[lmdb]") {
auto claim = std::make_shared<minifi::ResourceClaimImpl>(content_repo_);
writeContent(*claim);
REQUIRE(content_repo_->exists(*claim));
content_repo_->reset();
content_repo_->clearOrphans();
REQUIRE_FALSE(content_repo_->exists(*claim));
}
TEST_CASE_METHOD(LmdbContentRepositoryTests, "Empty repository reports zero entry count", "[lmdb]") {
REQUIRE(content_repo_->getRepositoryEntryCount() == 0);
}
TEST_CASE_METHOD(LmdbContentRepositoryTests, "Written value updates repository stats", "[lmdb]") {
auto claim = std::make_shared<minifi::ResourceClaimImpl>(content_repo_);
auto start_size = content_repo_->getRepositorySize();
writeContent(*claim);
REQUIRE(content_repo_->getRepositoryEntryCount() == 1);
auto end_size = content_repo_->getRepositorySize();
REQUIRE(end_size > start_size);
}
TEST_CASE("Content persists across LmdbContentRepository re-initialization", "[lmdb]") {
TestController controller;
auto db_path = controller.createTempDirectory();
auto configuration = std::make_shared<org::apache::nifi::minifi::ConfigureImpl>();
configuration->set(minifi::Configure::nifi_dbcontent_repository_directory_default, db_path.string());
std::string claim_path;
static constexpr std::string_view content = "persisted content";
{
auto content_repo = std::make_shared<core::repository::LmdbContentRepository>();
REQUIRE(content_repo->initialize(configuration));
auto claim = std::make_shared<minifi::ResourceClaimImpl>(content_repo);
claim_path = claim->getContentFullPath();
auto stream = content_repo->write(*claim, false);
stream->write(as_bytes(std::span(content)));
stream->close();
// ensure the content is not deleted on resource claim destruction
content_repo->incrementStreamCount(*claim);
}
auto reopened_repo = std::make_shared<core::repository::LmdbContentRepository>();
REQUIRE(reopened_repo->initialize(configuration));
auto reopened_claim = std::make_shared<minifi::ResourceClaimImpl>(claim_path, reopened_repo);
REQUIRE(reopened_repo->exists(*reopened_claim));
auto read_stream = reopened_repo->read(*reopened_claim);
std::vector<std::byte> buffer(content.size());
REQUIRE(read_stream->read(as_writable_bytes(std::span(buffer))) == content.size());
REQUIRE(std::string_view(reinterpret_cast<const char*>(buffer.data()), buffer.size()) == content);
}
TEST_CASE("ProcessSession::read reads the flowfile from offset to size", "[lmdb]") {
ContentRepositoryDependentTests::testReadOnSmallerClonedFlowFiles(std::make_shared<core::repository::LmdbContentRepository>());
}
TEST_CASE("ProcessSession::append should append to the flowfile and set its size correctly", "[lmdb]") {
ContentRepositoryDependentTests::testAppendToUnmanagedFlowFile(std::make_shared<core::repository::LmdbContentRepository>());
ContentRepositoryDependentTests::testAppendToManagedFlowFile(std::make_shared<core::repository::LmdbContentRepository>());
}
TEST_CASE("ProcessSession::read can read zero length flowfiles without crash", "[lmdb]") {
ContentRepositoryDependentTests::testReadFromZeroLengthFlowFile(std::make_shared<core::repository::LmdbContentRepository>());
}
TEST_CASE("ProcessSession::write can be cancelled", "[lmdb]") {
ContentRepositoryDependentTests::testOkWrite(std::make_shared<core::repository::LmdbContentRepository>());
ContentRepositoryDependentTests::testErrWrite(std::make_shared<core::repository::LmdbContentRepository>());
ContentRepositoryDependentTests::testCancelWrite(std::make_shared<core::repository::LmdbContentRepository>());
}
} // namespace org::apache::nifi::minifi::test