be/test/olap/memtable_memory_limiter_test.cpp

// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements.  See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership.  The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License.  You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied.  See the License for the
// specific language governing permissions and limitations
// under the License.

#include "olap/memtable_memory_limiter.h"

#include "exec/tablet_info.h"
#include "gtest/gtest_pred_impl.h"
#include "io/fs/local_file_system.h"
#include "olap/delta_writer.h"
#include "olap/storage_engine.h"
#include "olap/tablet_manager.h"
#include "runtime/descriptor_helper.h"
#include "runtime/descriptors.h"

namespace doris {
static const uint32_t MAX_PATH_LEN = 1024;

static void create_tablet_request(int64_t tablet_id, int32_t schema_hash,
                                  TCreateTabletReq* request) {
    request->tablet_id = tablet_id;
    request->__set_version(1);
    request->partition_id = 30002;
    request->tablet_schema.schema_hash = schema_hash;
    request->tablet_schema.short_key_column_count = 3;
    request->tablet_schema.keys_type = TKeysType::AGG_KEYS;
    request->tablet_schema.storage_type = TStorageType::COLUMN;
    request->__set_storage_format(TStorageFormat::V2);

    TColumn k1;
    k1.column_name = "k1";
    k1.__set_is_key(true);
    k1.column_type.type = TPrimitiveType::TINYINT;
    request->tablet_schema.columns.push_back(k1);

    TColumn k2;
    k2.column_name = "k2";
    k2.__set_is_key(true);
    k2.column_type.type = TPrimitiveType::SMALLINT;
    request->tablet_schema.columns.push_back(k2);

    TColumn k3;
    k3.column_name = "k3";
    k3.__set_is_key(true);
    k3.column_type.type = TPrimitiveType::INT;
    request->tablet_schema.columns.push_back(k3);
}

static TDescriptorTable create_descriptor_tablet() {
    TDescriptorTableBuilder dtb;
    TTupleDescriptorBuilder tuple_builder;

    tuple_builder.add_slot(
            TSlotDescriptorBuilder().type(TYPE_TINYINT).column_name("k1").column_pos(0).build());
    tuple_builder.add_slot(
            TSlotDescriptorBuilder().type(TYPE_SMALLINT).column_name("k2").column_pos(1).build());
    tuple_builder.add_slot(
            TSlotDescriptorBuilder().type(TYPE_INT).column_name("k3").column_pos(2).build());

    tuple_builder.build(&dtb);
    return dtb.desc_tbl();
}

class MemTableMemoryLimiterTest : public testing::Test {
protected:
    void SetUp() override {
        // set path
        char buffer[MAX_PATH_LEN];
        EXPECT_NE(getcwd(buffer, MAX_PATH_LEN), nullptr);
        config::storage_root_path = std::string(buffer) + "/data_test";
        auto st = io::global_local_filesystem()->delete_directory(config::storage_root_path);
        ASSERT_TRUE(st.ok()) << st;
        st = io::global_local_filesystem()->create_directory(config::storage_root_path);
        ASSERT_TRUE(st.ok()) << st;
        std::vector<StorePath> paths;
        paths.emplace_back(config::storage_root_path, -1);

        doris::EngineOptions options;
        options.store_paths = paths;
        auto engine = std::make_unique<StorageEngine>(options);
        _engine_ref = engine.get();
        st = engine->open();
        EXPECT_TRUE(st.ok()) << st.to_string();

        ExecEnv* exec_env = doris::ExecEnv::GetInstance();
        // ExecEnv's storage_engine will be read by storage_engine's other operations.
        // So we must do this before storage engine's other operation.
        exec_env->set_storage_engine(std::move(engine));
        exec_env->set_memtable_memory_limiter(new MemTableMemoryLimiter());
    }

    void TearDown() override {
        ExecEnv* exec_env = doris::ExecEnv::GetInstance();
        exec_env->set_memtable_memory_limiter(nullptr);
        _engine_ref = nullptr;
        exec_env->set_storage_engine(nullptr);
        EXPECT_EQ(system("rm -rf ./data_test"), 0);
        static_cast<void>(io::global_local_filesystem()->delete_directory(
                std::string(getenv("DORIS_HOME")) + "/" + UNUSED_PREFIX));
    }

    StorageEngine* _engine_ref = nullptr;
};

TEST_F(MemTableMemoryLimiterTest, handle_memtable_flush_test) {
    std::unique_ptr<RuntimeProfile> profile;
    profile = std::make_unique<RuntimeProfile>("CreateTablet");
    TCreateTabletReq request;
    create_tablet_request(10000, 270068372, &request);
    Status res = _engine_ref->create_tablet(request, profile.get());
    ASSERT_TRUE(res.ok());

    TDescriptorTable tdesc_tbl = create_descriptor_tablet();
    ObjectPool obj_pool;
    DescriptorTbl* desc_tbl = nullptr;
    static_cast<void>(DescriptorTbl::create(&obj_pool, tdesc_tbl, &desc_tbl));
    TupleDescriptor* tuple_desc = desc_tbl->get_tuple_descriptor(0);
    auto param = std::make_shared<OlapTableSchemaParam>();

    PUniqueId load_id;
    load_id.set_hi(0);
    load_id.set_lo(0);
    WriteRequest write_req;
    write_req.tablet_id = 10000;
    write_req.schema_hash = 270068372;
    write_req.txn_id = 20002;
    write_req.partition_id = 30002;
    write_req.load_id = load_id;
    write_req.tuple_desc = tuple_desc;
    write_req.slots = &(tuple_desc->slots());
    write_req.is_high_priority = false;
    write_req.table_schema_param = param;
    profile = std::make_unique<RuntimeProfile>("MemTableMemoryLimiterTest");
    auto delta_writer =
            std::make_unique<DeltaWriter>(*_engine_ref, write_req, profile.get(), TUniqueId {});
    ASSERT_NE(delta_writer, nullptr);
    auto mem_limiter = ExecEnv::GetInstance()->memtable_memory_limiter();

    vectorized::Block block;
    for (const auto& slot_desc : tuple_desc->slots()) {
        block.insert(vectorized::ColumnWithTypeAndName(slot_desc->get_empty_mutable_column(),
                                                       slot_desc->type(), slot_desc->col_name()));
    }
    auto columns = block.mutate_columns();
    {
        int8_t k1 = -127;
        columns[0]->insert_data((const char*)&k1, sizeof(k1));

        int16_t k2 = -32767;
        columns[1]->insert_data((const char*)&k2, sizeof(k2));

        int32_t k3 = -2147483647;
        columns[2]->insert_data((const char*)&k3, sizeof(k3));

        res = delta_writer->write(&block, {0});
        ASSERT_TRUE(res.ok());
    }
    static_cast<void>(mem_limiter->init(100));
    mem_limiter->handle_memtable_flush(nullptr);
    CHECK_EQ(0, mem_limiter->mem_usage());

    res = delta_writer->close();
    EXPECT_EQ(Status::OK(), res);
    res = delta_writer->build_rowset();
    EXPECT_EQ(Status::OK(), res);
    res = delta_writer->commit_txn(PSlaveTabletNodes());
    EXPECT_EQ(Status::OK(), res);
    res = _engine_ref->tablet_manager()->drop_tablet(request.tablet_id, request.replica_id, false);
    EXPECT_EQ(Status::OK(), res);
}
} // namespace doris