be/src/runtime/descriptors.h

// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements.  See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership.  The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License.  You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied.  See the License for the
// specific language governing permissions and limitations
// under the License.
// This file is copied from
// https://github.com/apache/impala/blob/branch-2.9.0/be/src/runtime/descriptors.h
// and modified by Doris

#pragma once

#include <gen_cpp/Descriptors_types.h>
#include <gen_cpp/Exprs_types.h>
#include <gen_cpp/Types_types.h>
#include <glog/logging.h>

#include <cstdint>
#include <ostream>
#include <string>
#include <unordered_map>
#include <utility>
#include <vector>

#include "common/be_mock_util.h"
#include "common/compiler_util.h" // IWYU pragma: keep
#include "common/global_types.h"
#include "common/object_pool.h"
#include "common/status.h"
#include "core/data_type/data_type.h"
#include "core/data_type/define_primitive_type.h"
#include "storage/utils.h"

namespace doris {
class ObjectPool;
class PTupleDescriptor;
class PSlotDescriptor;

using TColumnAccessPaths = std::vector<TColumnAccessPath>;

class SlotDescriptor {
public:
    MOCK_DEFINE(virtual ~SlotDescriptor() = default;)
    SlotId id() const { return _id; }
    DataTypePtr type() const { return _type; }
    TupleId parent() const { return _parent; }
    // Returns the column index of this slot, including partition keys.
    // (e.g., col_pos - num_partition_keys = the table column this slot corresponds to)
    int col_pos() const { return _col_pos; }
    // Returns the field index in the generated llvm struct for this slot's tuple
    int field_idx() const { return _field_idx; }
    bool is_nullable() const;
    DataTypePtr get_data_type_ptr() const;

    const std::string& col_name() const { return _col_name; }
    const std::string& col_name_lower_case() const { return _col_name_lower_case; }

    void to_protobuf(PSlotDescriptor* pslot) const;

    std::string debug_string() const;

    MutableColumnPtr get_empty_mutable_column() const;

    MOCK_FUNCTION int32_t col_unique_id() const { return _col_unique_id; }

    bool is_key() const { return _is_key; }
    const std::vector<std::string>& column_paths() const { return _column_paths; };

    const TColumnAccessPaths& all_access_paths() const { return _all_access_paths; }
    const TColumnAccessPaths& predicate_access_paths() const { return _predicate_access_paths; }

    bool is_auto_increment() const { return _is_auto_increment; }

    bool is_skip_bitmap_col() const { return _col_name == SKIP_BITMAP_COL; }
    bool is_sequence_col() const { return _col_name == SEQUENCE_COL; }

    const std::string& col_default_value() const { return _col_default_value; }
    PrimitiveType col_type() const;

    std::shared_ptr<doris::TExpr> get_virtual_column_expr() const {
        // virtual_column_expr need do prepare.
        return virtual_column_expr;
    }

    void set_is_predicate(bool is_predicate) { _is_predicate = is_predicate; }

    bool is_predicate() const { return _is_predicate; }

private:
    friend class DescriptorTbl;
    friend class TupleDescriptor;
    friend class SchemaScanner;
    friend class OlapTableSchemaParam;
    friend class PInternalServiceImpl;
    friend class RowIdStorageReader;
    friend class Tablet;
    friend class TabletSchema;

    MOCK_REMOVE(const) SlotId _id;
    MOCK_REMOVE(const) DataTypePtr _type;
    const TupleId _parent;
    const int _col_pos;
    MOCK_REMOVE(const) std::string _col_name;
    const std::string _col_name_lower_case;

    const int32_t _col_unique_id;

    // the idx of the slot in the tuple descriptor (0-based).
    // this is provided by the FE
    const int _slot_idx;

    // the idx of the slot in the llvm codegen'd tuple struct
    // this is set by TupleDescriptor during codegen and takes into account
    // leading null bytes.
    int _field_idx;

    const bool _is_key;
    const std::vector<std::string> _column_paths;

    TColumnAccessPaths _all_access_paths;
    TColumnAccessPaths _predicate_access_paths;

    const bool _is_auto_increment;
    const std::string _col_default_value;

    std::shared_ptr<doris::TExpr> virtual_column_expr = nullptr;

    bool _is_predicate = false;

    SlotDescriptor(const TSlotDescriptor& tdesc);
    SlotDescriptor(const PSlotDescriptor& pdesc);
    MOCK_DEFINE(SlotDescriptor();)
};

// Base class for table descriptors.
class TableDescriptor {
public:
    TableDescriptor(const TTableDescriptor& tdesc);
    virtual ~TableDescriptor() = default;
    int num_cols() const { return _num_cols; }
    int num_clustering_cols() const { return _num_clustering_cols; }
    virtual std::string debug_string() const;

    // The first _num_clustering_cols columns by position are clustering
    // columns.
    bool is_clustering_col(const SlotDescriptor* slot_desc) const {
        return slot_desc->col_pos() < _num_clustering_cols;
    }

    ::doris::TTableType::type table_type() const { return _table_type; }
    const std::string& name() const { return _name; }
    const std::string& database() const { return _database; }
    int64_t table_id() const { return _table_id; }

private:
    ::doris::TTableType::type _table_type;
    std::string _name;
    std::string _database;
    int64_t _table_id;
    int _num_cols;
    int _num_clustering_cols;
};

class OlapTableDescriptor : public TableDescriptor {
public:
    OlapTableDescriptor(const TTableDescriptor& tdesc);
    std::string debug_string() const override;
};

class DictionaryTableDescriptor : public TableDescriptor {
public:
    DictionaryTableDescriptor(const TTableDescriptor& tdesc);
    std::string debug_string() const override;
};

class SchemaTableDescriptor : public TableDescriptor {
public:
    SchemaTableDescriptor(const TTableDescriptor& tdesc);
    ~SchemaTableDescriptor() override;
    std::string debug_string() const override;
    TSchemaTableType::type schema_table_type() const { return _schema_table_type; }

private:
    TSchemaTableType::type _schema_table_type;
};

class BrokerTableDescriptor : public TableDescriptor {
public:
    BrokerTableDescriptor(const TTableDescriptor& tdesc);
    ~BrokerTableDescriptor() override;
    std::string debug_string() const override;

private:
};

class HiveTableDescriptor : public TableDescriptor {
public:
    HiveTableDescriptor(const TTableDescriptor& tdesc);
    ~HiveTableDescriptor() override;
    std::string debug_string() const override;

private:
};

class IcebergTableDescriptor : public TableDescriptor {
public:
    IcebergTableDescriptor(const TTableDescriptor& tdesc);
    ~IcebergTableDescriptor() override;
    std::string debug_string() const override;

private:
};

class MaxComputeTableDescriptor : public TableDescriptor {
public:
    MaxComputeTableDescriptor(const TTableDescriptor& tdesc);
    ~MaxComputeTableDescriptor() override;
    std::string debug_string() const override;
    std::string region() const { return _region; }
    std::string project() const { return _project; }
    std::string table() const { return _table; }
    std::string odps_url() const { return _odps_url; }
    std::string tunnel_url() const { return _tunnel_url; }
    std::string access_key() const { return _access_key; }
    std::string secret_key() const { return _secret_key; }
    std::string public_access() const { return _public_access; }
    std::string endpoint() const { return _endpoint; }
    std::string quota() const { return _quota; }
    Status init_status() const { return _init_status; }
    std::map<std::string, std::string> properties() const { return _props; }

private:
    std::string _region; //deprecated
    std::string _project;
    std::string _table;
    std::string _odps_url;      //deprecated
    std::string _tunnel_url;    //deprecated
    std::string _access_key;    //deprecated
    std::string _secret_key;    //deprecated
    std::string _public_access; //deprecated
    std::string _endpoint;
    std::string _quota;
    std::map<std::string, std::string> _props;
    Status _init_status = Status::OK();
};

class TrinoConnectorTableDescriptor : public TableDescriptor {
public:
    TrinoConnectorTableDescriptor(const TTableDescriptor& tdesc);
    ~TrinoConnectorTableDescriptor() override;
    std::string debug_string() const override;

private:
};

class EsTableDescriptor : public TableDescriptor {
public:
    EsTableDescriptor(const TTableDescriptor& tdesc);
    ~EsTableDescriptor() override;
    std::string debug_string() const override;

private:
};

class MySQLTableDescriptor : public TableDescriptor {
public:
    MySQLTableDescriptor(const TTableDescriptor& tdesc);
    std::string debug_string() const override;
    std::string mysql_db() const { return _mysql_db; }
    std::string mysql_table() const { return _mysql_table; }
    std::string host() const { return _host; }
    std::string port() const { return _port; }
    std::string user() const { return _user; }
    std::string passwd() const { return _passwd; }
    std::string charset() const { return _charset; }

private:
    std::string _mysql_db;
    std::string _mysql_table;
    std::string _host;
    std::string _port;
    std::string _user;
    std::string _passwd;
    std::string _charset;
};

class JdbcTableDescriptor : public TableDescriptor {
public:
    JdbcTableDescriptor(const TTableDescriptor& tdesc);
    std::string debug_string() const override;
    int64_t jdbc_catalog_id() const { return _jdbc_catalog_id; }
    const std::string& jdbc_resource_name() const { return _jdbc_resource_name; }
    const std::string& jdbc_driver_url() const { return _jdbc_driver_url; }
    const std::string& jdbc_driver_class() const { return _jdbc_driver_class; }
    const std::string& jdbc_driver_checksum() const { return _jdbc_driver_checksum; }
    const std::string& jdbc_url() const { return _jdbc_url; }
    const std::string& jdbc_table_name() const { return _jdbc_table_name; }
    const std::string& jdbc_user() const { return _jdbc_user; }
    const std::string& jdbc_passwd() const { return _jdbc_passwd; }
    int32_t connection_pool_min_size() const { return _connection_pool_min_size; }
    int32_t connection_pool_max_size() const { return _connection_pool_max_size; }
    int32_t connection_pool_max_wait_time() const { return _connection_pool_max_wait_time; }
    int32_t connection_pool_max_life_time() const { return _connection_pool_max_life_time; }
    bool connection_pool_keep_alive() const { return _connection_pool_keep_alive; }

private:
    int64_t _jdbc_catalog_id;
    std::string _jdbc_resource_name;
    std::string _jdbc_driver_url;
    std::string _jdbc_driver_class;
    std::string _jdbc_driver_checksum;
    std::string _jdbc_url;
    std::string _jdbc_table_name;
    std::string _jdbc_user;
    std::string _jdbc_passwd;
    int32_t _connection_pool_min_size;
    int32_t _connection_pool_max_size;
    int32_t _connection_pool_max_wait_time;
    int32_t _connection_pool_max_life_time;
    bool _connection_pool_keep_alive;
};

class RemoteDorisTableDescriptor : public TableDescriptor {
public:
    RemoteDorisTableDescriptor(const TTableDescriptor& tdesc);
    ~RemoteDorisTableDescriptor() override;
    std::string debug_string() const override;

private:
};

class TupleDescriptor {
public:
    TupleDescriptor(TupleDescriptor&&) = delete;
    void operator=(const TupleDescriptor&) = delete;

    MOCK_DEFINE(virtual) ~TupleDescriptor() {
        if (_own_slots) {
            for (SlotDescriptor* slot : _slots) {
                delete slot;
            }
        }
    }

    MOCK_DEFINE(TupleDescriptor() : _id {0} {};)

    int num_materialized_slots() const { return _num_materialized_slots; }
    MOCK_FUNCTION const std::vector<SlotDescriptor*>& slots() const { return _slots; }
    int get_column_id(SlotId slot_id) const;

    bool has_varlen_slots() const { return _has_varlen_slots; }
    const TableDescriptor* table_desc() const { return _table_desc; }

    TupleId id() const { return _id; }

    std::string debug_string() const;

    void to_protobuf(PTupleDescriptor* ptuple) const;

private:
    friend class DescriptorTbl;
    friend class SchemaScanner;
    friend class OlapTableSchemaParam;
    friend class PInternalServiceImpl;
    friend class RowIdStorageReader;
    friend class TabletSchema;

    const TupleId _id;
    TableDescriptor* _table_desc = nullptr;
    int _num_materialized_slots;
    std::vector<SlotDescriptor*> _slots; // contains all slots

    // Provide quick way to check if there are variable length slots.
    // True if _string_slots or _collection_slots have entries.
    bool _has_varlen_slots;
    bool _own_slots = false;

    TupleDescriptor(const TTupleDescriptor& tdesc, bool own_slot = false);
    TupleDescriptor(const PTupleDescriptor& tdesc, bool own_slot = false);

    void add_slot(SlotDescriptor* slot);
};

class DescriptorTbl {
public:
#ifdef BE_TEST
    DescriptorTbl() = default;
    virtual ~DescriptorTbl() = default;
#endif

    // Creates a descriptor tbl within 'pool' from thrift_tbl and returns it via 'tbl'.
    // Returns OK on success, otherwise error (in which case 'tbl' will be unset).
    static Status create(ObjectPool* pool, const TDescriptorTable& thrift_tbl, DescriptorTbl** tbl);

    TableDescriptor* get_table_descriptor(TableId id) const;
    TupleDescriptor* get_tuple_descriptor(TupleId id) const;
    MOCK_FUNCTION SlotDescriptor* get_slot_descriptor(SlotId id) const;
    const std::vector<TTupleId>& get_row_tuples() const { return _row_tuples; }

    // return all registered tuple descriptors
    std::vector<TupleDescriptor*> get_tuple_descs() const {
        std::vector<TupleDescriptor*> descs;

        for (auto it : _tuple_desc_map) {
            descs.push_back(it.second);
        }

        return descs;
    }

    std::string debug_string() const;

private:
    using TableDescriptorMap = std::unordered_map<TableId, TableDescriptor*>;
    using TupleDescriptorMap = std::unordered_map<TupleId, TupleDescriptor*>;
    using SlotDescriptorMap = std::unordered_map<SlotId, SlotDescriptor*>;

    TableDescriptorMap _tbl_desc_map;
    TupleDescriptorMap _tuple_desc_map;
    SlotDescriptorMap _slot_desc_map;
    std::vector<TTupleId> _row_tuples;

#ifndef BE_TEST
    DescriptorTbl() = default;
#endif
};

// Describes the ordered tuples whose slots form an operator's block layout.
class RowDescriptor {
public:
    RowDescriptor(const DescriptorTbl& desc_tbl, const std::vector<TTupleId>& row_tuples);

    RowDescriptor(TupleDescriptor* tuple_desc);

    // Empty layout for expressions that do not reference slots.
    RowDescriptor() = default;

    MOCK_DEFINE(virtual ~RowDescriptor() = default;)

    int num_slots() const;

    // Return descriptors for all tuples in this row, in order of appearance.
    MOCK_FUNCTION const std::vector<TupleDescriptor*>& tuple_descriptors() const {
        return _tuple_desc_map;
    }

    std::string debug_string() const;

    int get_column_id(int slot_id) const;

private:
    // Tuples in block column order; descriptors are owned elsewhere.
    std::vector<TupleDescriptor*> _tuple_desc_map;
};
} // namespace doris