gensrc/thrift/PaloInternalService.thrift

// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements.  See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership.  The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License.  You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied.  See the License for the
// specific language governing permissions and limitations
// under the License.

namespace cpp doris
namespace java org.apache.doris.thrift

include "Status.thrift"
include "Types.thrift"
include "Exprs.thrift"
include "Descriptors.thrift"
include "PlanNodes.thrift"
include "Planner.thrift"
include "DataSinks.thrift"
include "Data.thrift"
include "RuntimeProfile.thrift"
include "PaloService.thrift"

// constants for TQueryOptions.num_nodes
const i32 NUM_NODES_ALL = 0
const i32 NUM_NODES_ALL_RACKS = -1

// constants for TPlanNodeId
const i32 INVALID_PLAN_NODE_ID = -1

// Constant default partition ID, must be < 0 to avoid collisions
const i64 DEFAULT_PARTITION_ID = -1;

enum TQueryType {
    SELECT,
    LOAD,
    EXTERNAL
}

enum TErrorHubType {
    MYSQL,
    BROKER,
    NULL_TYPE
}

enum TPrefetchMode {
    NONE,
    HT_BUCKET
}

struct TMysqlErrorHubInfo {
    1: required string host;
    2: required i32 port;
    3: required string user;
    4: required string passwd;
    5: required string db;
    6: required string table;
}

struct TBrokerErrorHubInfo {
    1: required Types.TNetworkAddress broker_addr;
    2: required string path;
    3: required map<string, string> prop;
}

struct TLoadErrorHubInfo {
    1: required TErrorHubType type = TErrorHubType.NULL_TYPE;
    2: optional TMysqlErrorHubInfo mysql_info;
    3: optional TBrokerErrorHubInfo broker_info;
}

struct TResourceLimit {
    1: optional i32 cpu_limit
}

// Query options that correspond to PaloService.PaloQueryOptions,
// with their respective defaults
struct TQueryOptions {
  1: optional bool abort_on_error = 0
  2: optional i32 max_errors = 0
  3: optional bool disable_codegen = 1
  4: optional i32 batch_size = 0
  5: optional i32 num_nodes = NUM_NODES_ALL
  6: optional i64 max_scan_range_length = 0
  7: optional i32 num_scanner_threads = 0
  8: optional i32 max_io_buffers = 0
  9: optional bool allow_unsupported_formats = 0
  10: optional i64 default_order_by_limit = -1
  // 11: optional string debug_action = "" // Never used
  12: optional i64 mem_limit = 2147483648
  13: optional bool abort_on_default_limit_exceeded = 0
  14: optional i32 query_timeout = 3600
  15: optional bool is_report_success = 0
  16: optional i32 codegen_level = 0
  // INT64::MAX
  17: optional i64 kudu_latest_observed_ts = 9223372036854775807 // Deprecated
  18: optional TQueryType query_type = TQueryType.SELECT
  19: optional i64 min_reservation = 0
  20: optional i64 max_reservation = 107374182400
  21: optional i64 initial_reservation_total_claims = 2147483647 // TODO chenhao
  22: optional i64 buffer_pool_limit = 2147483648

  // The default spillable buffer size in bytes, which may be overridden by the planner.
  // Defaults to 2MB.
  23: optional i64 default_spillable_buffer_size = 2097152;

  // The minimum spillable buffer to use. The planner will not choose a size smaller than
  // this. Defaults to 64KB.
  24: optional i64 min_spillable_buffer_size = 65536;

  // The maximum size of row that the query will reserve memory to process. Processing
  // rows larger than this may result in a query failure. Defaults to 512KB, e.g.
  // enough for a row with 15 32KB strings or many smaller columns.
  //
  // Different operators handle this option in different ways. E.g. some simply increase
  // the size of all their buffers to fit this row size, whereas others may use more
  // sophisticated strategies - e.g. reserving a small number of buffers large enough to
  // fit maximum-sized rows.
  25: optional i64 max_row_size = 524288;

  // stream preaggregation
  26: optional bool disable_stream_preaggregations = false;

  // multithreaded degree of intra-node parallelism
  27: optional i32 mt_dop = 0;
  // if this is a query option for LOAD, load_mem_limit should be set to limit the mem comsuption
  // of load channel.
  28: optional i64 load_mem_limit = 0;
  // see BE config `doris_max_scan_key_num` for details
  // if set, this will overwrite the BE config.
  29: optional i32 max_scan_key_num;
  // see BE config `max_pushdown_conditions_per_column` for details
  // if set, this will overwrite the BE config.
  30: optional i32 max_pushdown_conditions_per_column
  // whether enable spilling to disk
  31: optional bool enable_spilling = false;
  // whether enable parallel merge in exchange node
  32: optional bool enable_enable_exchange_node_parallel_merge = false;

  // Time in ms to wait until runtime filters are delivered.
  33: optional i32 runtime_filter_wait_time_ms = 1000

  // if the right table is greater than this value in the hash join,  we will ignore IN filter
  34: optional i32 runtime_filter_max_in_num = 1024;

  // whether enable vectorized engine 
  41: optional bool enable_vectorized_engine = false

  // the resource limitation of this query
  42: optional TResourceLimit resource_limit

  // show bitmap data in result, if use this in mysql cli may make the terminal
  // output corrupted character
  43: optional bool return_object_data_as_binary = false

  // trim tailing spaces while querying external table and stream load
  44: optional bool trim_tailing_spaces_for_external_table_query = false

  45: optional bool enable_function_pushdown;

  46: optional string fragment_transmission_compression_codec;

  48: optional bool enable_local_exchange;

  // For debug purpose, dont' merge unique key and agg key when reading data.
  49: optional bool skip_storage_engine_merge = false

  // For debug purpose, skip delete predicates when reading data
  50: optional bool skip_delete_predicate = false

  51: optional bool enable_new_shuffle_hash_method

  52: optional i32 be_exec_version = 0
  
  53: optional i32 partitioned_hash_join_rows_threshold = 0

  54: optional bool enable_share_hash_table_for_broadcast_join

  55: optional bool check_overflow_for_decimal = false

  // For debug purpose, skip delete bitmap when reading data
  56: optional bool skip_delete_bitmap = false
}
    

// A scan range plus the parameters needed to execute that scan.
struct TScanRangeParams {
  1: required PlanNodes.TScanRange scan_range
  2: optional i32 volume_id = -1
}

// Specification of one output destination of a plan fragment
struct TPlanFragmentDestination {
  // the globally unique fragment instance id
  1: required Types.TUniqueId fragment_instance_id

  // ... which is being executed on this server
  2: required Types.TNetworkAddress server
  3: optional Types.TNetworkAddress brpc_server
}

struct TRuntimeFilterTargetParams {
  1: required Types.TUniqueId target_fragment_instance_id
  // The address of the instance where the fragment is expected to run
  2: required Types.TNetworkAddress target_fragment_instance_addr
}

struct TRuntimeFilterParams {
  // Runtime filter merge instance address
  1: optional Types.TNetworkAddress runtime_filter_merge_addr

  // Runtime filter ID to the instance address of the fragment,
  // that is expected to use this runtime filter
  2: optional map<i32, list<TRuntimeFilterTargetParams>> rid_to_target_param

  // Runtime filter ID to the runtime filter desc
  3: optional map<i32, PlanNodes.TRuntimeFilterDesc> rid_to_runtime_filter

  // Number of Runtime filter producers
  4: optional map<i32, i32> runtime_filter_builder_num
}

// Parameters for a single execution instance of a particular TPlanFragment
// TODO: for range partitioning, we also need to specify the range boundaries
struct TPlanFragmentExecParams {
  // a globally unique id assigned to the entire query
  1: required Types.TUniqueId query_id

  // a globally unique id assigned to this particular execution instance of
  // a TPlanFragment
  2: required Types.TUniqueId fragment_instance_id

  // initial scan ranges for each scan node in TPlanFragment.plan_tree
  3: required map<Types.TPlanNodeId, list<TScanRangeParams>> per_node_scan_ranges

  // number of senders for ExchangeNodes contained in TPlanFragment.plan_tree;
  // needed to create a DataStreamRecvr
  4: required map<Types.TPlanNodeId, i32> per_exch_num_senders

  // Output destinations, one per output partition.
  // The partitioning of the output is specified by
  // TPlanFragment.output_sink.output_partition.
  // The number of output partitions is destinations.size().
  5: list<TPlanFragmentDestination> destinations

  // Debug options: perform some action in a particular phase of a particular node
  // 6: optional Types.TPlanNodeId debug_node_id // Never used
  // 7: optional PlanNodes.TExecNodePhase debug_phase // Never used
  // 8: optional PlanNodes.TDebugAction debug_action // Never used

  // Id of this fragment in its role as a sender.
  9: optional i32 sender_id
  10: optional i32 num_senders
  11: optional bool send_query_statistics_with_every_batch
  // Used to merge and send runtime filter
  12: optional TRuntimeFilterParams runtime_filter_params
}

// Global query parameters assigned by the coordinator.
struct TQueryGlobals {
  // String containing a timestamp set as the current time.
  // Format is yyyy-MM-dd HH:mm:ss
  1: required string now_string

  // To support timezone in Doris. timestamp_ms is the millisecond uinix timestamp for
  // this query to calculate time zone relative function 
  2: optional i64 timestamp_ms

  // time_zone is the timezone this query used.
  // If this value is set, BE will ignore now_string
  3: optional string time_zone

  // Set to true if in a load plan, the max_filter_ratio is 0.0
  4: optional bool load_zero_tolerance = false

  5: optional i32 nano_seconds
}


// Service Protocol Details

enum PaloInternalServiceVersion {
  V1
}

struct TTxnParams {
  1: optional bool need_txn
  2: optional string auth_code_uuid
  3: optional i64 thrift_rpc_timeout_ms
  4: optional string db
  5: optional string tbl
  6: optional string user_ip
  7: optional i64 txn_id
  8: optional Types.TUniqueId fragment_instance_id
  9: optional i64 db_id
  10: optional double max_filter_ratio
}

// Definition of global dict, global dict is used to accelerate query performance of low cardinality data
struct TColumnDict {
  1: optional Types.TPrimitiveType type
  2: list<string> str_dict  // map one string to a integer, using offset as id
}

struct TGlobalDict {
  1: optional map<i32, TColumnDict> dicts,  // map dict_id to column dict
  2: optional map<i32, i32> slot_dicts // map from slot id to column dict id, because 2 or more column may share the dict
}

// ExecPlanFragment
struct TExecPlanFragmentParams {
  1: required PaloInternalServiceVersion protocol_version

  // required in V1
  2: optional Planner.TPlanFragment fragment

  // required in V1
  // @Common components
  3: optional Descriptors.TDescriptorTable desc_tbl

  // required in V1
  4: optional TPlanFragmentExecParams params

  // Initiating coordinator.
  // TODO: determine whether we can get this somehow via the Thrift rpc mechanism.
  // required in V1
  // @Common components
  5: optional Types.TNetworkAddress coord

  // backend number assigned by coord to identify backend
  // required in V1
  6: optional i32 backend_num

  // Global query parameters assigned by coordinator.
  // required in V1
  // @Common components
  7: optional TQueryGlobals query_globals

  // options for the query
  // required in V1
  8: optional TQueryOptions query_options

  // Whether reportd when the backend fails
  // required in V1
  9: optional bool is_report_success

  // required in V1
  // @Common components
  10: optional Types.TResourceInfo resource_info

  // load job related
  11: optional string import_label
  12: optional string db_name
  13: optional i64 load_job_id
  14: optional TLoadErrorHubInfo load_error_hub_info

  // The total number of fragments on same BE host
  15: optional i32 fragment_num_on_host

  // If true, all @Common components is unset and should be got from BE's cache
  // If this field is unset or it set to false, all @Common components is set.
  16: optional bool is_simplified_param = false;
  17: optional TTxnParams txn_conf
  18: optional i64 backend_id
  19: optional TGlobalDict global_dict  // scan node could use the global dict to encode the string value to an integer

  // If it is true, after this fragment is prepared on the BE side,
  // it will wait for the FE to send the "start execution" command before it is actually executed.
  // Otherwise, the fragment will start executing directly on the BE side.
  20: optional bool need_wait_execution_trigger = false;
}

struct TExecPlanFragmentParamsList {
    1: optional list<TExecPlanFragmentParams> paramsList;
}

struct TExecPlanFragmentResult {
  // required in V1
  1: optional Status.TStatus status
}

// CancelPlanFragment
struct TCancelPlanFragmentParams {
  1: required PaloInternalServiceVersion protocol_version

  // required in V1
  2: optional Types.TUniqueId fragment_instance_id
}

struct TCancelPlanFragmentResult {
  // required in V1
  1: optional Status.TStatus status
}

// fold constant expr
struct TExprMap {
  1: required map<string, Exprs.TExpr> expr_map
}

struct TFoldConstantParams {
  1: required map<string, map<string, Exprs.TExpr>> expr_map
  2: required TQueryGlobals query_globals
  3: optional bool vec_exec
}

// TransmitData
struct TTransmitDataParams {
  1: required PaloInternalServiceVersion protocol_version

  // required in V1
  2: optional Types.TUniqueId dest_fragment_instance_id

  // for debugging purposes; currently ignored
  //3: optional Types.TUniqueId src_fragment_instance_id

  // required in V1
  4: optional Types.TPlanNodeId dest_node_id

  // required in V1
  5: optional Data.TRowBatch row_batch

  // if set to true, indicates that no more row batches will be sent
  // for this dest_node_id
  6: optional bool eos

  7: optional i32 be_number
  8: optional i64 packet_seq

  // Id of this fragment in its role as a sender.
  9: optional i32 sender_id
}

struct TTransmitDataResult {
  // required in V1
  1: optional Status.TStatus status
  2: optional i64 packet_seq
  3: optional Types.TUniqueId dest_fragment_instance_id
  4: optional Types.TPlanNodeId dest_node_id
}

struct TTabletWithPartition {
    1: required i64 partition_id
    2: required i64 tablet_id
}

// open a tablet writer
struct TTabletWriterOpenParams {
    1: required Types.TUniqueId id
    2: required i64 index_id
    3: required i64 txn_id
    4: required Descriptors.TOlapTableSchemaParam schema
    5: required list<TTabletWithPartition> tablets

    6: required i32 num_senders
}

struct TTabletWriterOpenResult {
    1: required Status.TStatus status
}

// add batch to tablet writer
struct TTabletWriterAddBatchParams {
    1: required Types.TUniqueId id
    2: required i64 index_id

    3: required i64 packet_seq
    4: required list<Types.TTabletId> tablet_ids
    5: required Data.TRowBatch row_batch

    6: required i32 sender_no
}

struct TTabletWriterAddBatchResult {
    1: required Status.TStatus status
}

struct TTabletWriterCloseParams {
    1: required Types.TUniqueId id
    2: required i64 index_id

    3: required i32 sender_no
}

struct TTabletWriterCloseResult {
    1: required Status.TStatus status
}

//
struct TTabletWriterCancelParams {
    1: required Types.TUniqueId id
    2: required i64 index_id

    3: required i32 sender_no
}

struct TTabletWriterCancelResult {
}

struct TFetchDataParams {
  1: required PaloInternalServiceVersion protocol_version
  // required in V1
  // query id which want to fetch data
  2: required Types.TUniqueId fragment_instance_id
}

struct TFetchDataResult {
    // result batch
    1: required Data.TResultBatch result_batch
    // end of stream flag
    2: required bool eos
    // packet num used check lost of packet
    3: required i32 packet_num
    // Operation result
    4: optional Status.TStatus status
}

struct TCondition {
    1:  required string column_name
    2:  required string condition_op
    3:  required list<string> condition_values
    // In delete condition, the different column may have same column name, need
    // using unique id to distinguish them
    4:  optional i32 column_unique_id
}

struct TExportStatusResult {
    1: required Status.TStatus status
    2: required Types.TExportState state
    3: optional list<string> files
}