gensrc/proto/internal_service.proto

// Licensed to the Apache Software Foundation (ASF) under one

// or more contributor license agreements.  See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership.  The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License.  You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied.  See the License for the
// specific language governing permissions and limitations
// under the License.

syntax="proto2";

package doris;
option java_package = "org.apache.doris.proto";

import "data.proto";
import "descriptors.proto";
import "types.proto";
import "olap_common.proto";
import "olap_file.proto";

option cc_generic_services = true;

// Transmit data when process SQL query
message PTransmitDataParams {
    // non-change member
    required PUniqueId finst_id = 1;
    required int32 node_id = 2;
    // Id of this fragment in its role as a sender.
    required int32 sender_id = 3;
    required int32 be_number = 4;

    // if set to true, indicates that no more row batches will be sent
    // for this dest_node_id
    required bool eos = 5;
    optional PRowBatch row_batch = 6;
    // different per packet
    required int64 packet_seq = 7;
    optional PQueryStatistics query_statistics = 8;

    optional PBlock block = 9;
    // transfer the RowBatch to the Controller Attachment
    optional bool transfer_by_attachment = 10 [default = false];
    optional PUniqueId query_id = 11;
    optional PStatus exec_status = 12;
};

message PTransmitDataResult {
    optional PStatus status = 1;
    optional int64 receive_time = 2;
};

message PTabletWithPartition {
    required int64 partition_id = 1;
    required int64 tablet_id = 2;
}

message PTabletLoadRowsetInfo {
    required int32 current_rowset_nums = 1;
    required int32 max_config_rowset_nums = 2;
}

message PTabletID {
    optional int64 partition_id = 1;
    optional int64 index_id = 2;
    optional int64 tablet_id = 3;
    optional int64 num_segments = 4;
}

message PTabletInfo {
    required int64 tablet_id = 1;
    required int32 schema_hash = 2;
    // Delta Writer will write data to local disk and then check if there are new raw values not in global dict
    // if appears, then it should add the column name to this vector
    repeated string invalid_dict_cols = 3; 
    // total rows num received by DeltaWriter
    optional int64 received_rows = 4;
    optional int64 num_rows_filtered = 5 [default = 0];
}

// open a tablet writer
message PTabletWriterOpenRequest {
    required PUniqueId id = 1;
    required int64 index_id = 2;
    required int64 txn_id = 3;
    required POlapTableSchemaParam schema = 4;
    repeated PTabletWithPartition tablets = 5;
    required int32 num_senders = 6;
    required bool need_gen_rollup = 7;  //Deprecated, not used since alter job v2
    optional int64 load_mem_limit = 8;
    optional int64 load_channel_timeout_s = 9;
    optional bool is_high_priority = 10 [default = false];
    optional string sender_ip = 11 [default = ""];
    optional bool is_vectorized = 12 [default = false];
    optional int64 backend_id = 13 [default = -1];
    optional bool enable_profile = 14 [default = false];
    optional bool is_incremental = 15 [default = false];
    optional int64 txn_expiration = 16; // Absolute time
    optional bool write_file_cache = 17;
    optional string storage_vault_id = 18;
    optional int32 sender_id = 19;
    optional int64 workload_group_id = 20;
};

message PTabletWriterOpenResult {
    required PStatus status = 1;
    repeated PTabletLoadRowsetInfo tablet_load_rowset_num_infos = 2;
};

// add batch to tablet writer
message PTabletWriterAddBatchRequest {
    required PUniqueId id = 1;
    required int64 index_id = 2;
    required int32 sender_id = 3;

    // If this is the last batch from this sender
    optional bool eos = 4;

    required int64 packet_seq = 5;
    repeated int64 tablet_ids = 6;
    // unset if and only if when eos is true
    optional PRowBatch row_batch = 7;
    // only valid when eos is true
    // valid partition ids that would write in this writer
    repeated int64 partition_ids = 8;
    // the backend which send this request
    optional int64 backend_id = 9 [default = -1];
    // transfer the RowBatch to the Controller Attachment
    optional bool transfer_by_attachment = 10 [default = false];
    optional bool is_high_priority = 11 [default = false];
    optional bool write_single_replica = 12 [default = false];
    map<int64, PSlaveTabletNodes> slave_tablet_nodes = 13;
};

message PTabletWriterAddBlockRequest {
    required PUniqueId id = 1;
    required int64 index_id = 2;
    required int32 sender_id = 3;

    // If this is the last batch from this sender
    optional bool eos = 4;

    optional int64 packet_seq = 5;
    repeated int64 tablet_ids = 6;
    // unset if and only if when eos is true
    optional PBlock block = 7;
    // only valid when eos is true
    // valid partition ids that would write in this writer
    repeated int64 partition_ids = 8;
    // the backend which send this request
    optional int64 backend_id = 9 [default = -1];
    // transfer the vectorized::Block to the Controller Attachment
    optional bool transfer_by_attachment = 10 [default = false];
    optional bool is_high_priority = 11 [default = false];
    optional bool write_single_replica = 12 [default = false];
    map<int64, PSlaveTabletNodes> slave_tablet_nodes = 13;
    optional bool is_single_tablet_block = 14 [default = false];
    // for auto-partition first stage close, we should hang.
    optional bool hang_wait = 15 [default = false];
};

message PSlaveTabletNodes {
    repeated PNodeInfo slave_nodes = 1;
}

message PNodeInfo {
    optional int64 id = 1;
    optional int64 option = 2;
    optional string host = 3;
    // used to transfer data between nodes
    optional int32 async_internal_port = 4;
}

message PSuccessSlaveTabletNodeIds {
    repeated int64 slave_node_ids = 1;
}

message PTabletError {
    optional int64 tablet_id = 1;
    optional string msg = 2;
}

message PTabletWriterAddBatchResult {
    required PStatus status = 1;
    repeated PTabletInfo tablet_vec = 2;
    optional int64 execution_time_us = 3;
    optional int64 wait_lock_time_us = 4;
    optional int64 wait_execution_time_us = 5;
    repeated PTabletError tablet_errors = 6;
    map<int64, PSuccessSlaveTabletNodeIds> success_slave_tablet_node_ids = 7;

    // For cloud
    optional int64 build_rowset_latency_ms = 1000;
    optional int64 commit_rowset_latency_ms = 1001;
    repeated PTabletLoadRowsetInfo tablet_load_rowset_num_infos = 1002;
};

message PTabletWriterAddBlockResult {
    required PStatus status = 1;
    repeated PTabletInfo tablet_vec = 2;
    optional int64 execution_time_us = 3;
    optional int64 wait_lock_time_us = 4;
    optional int64 wait_execution_time_us = 5;
    repeated PTabletError tablet_errors = 6;
    map<int64, PSuccessSlaveTabletNodeIds> success_slave_tablet_node_ids = 7;
    optional bytes load_channel_profile = 8;

    // For cloud
    optional int64 build_rowset_latency_ms = 1000;
    optional int64 commit_rowset_latency_ms = 1001;
    repeated PTabletLoadRowsetInfo tablet_load_rowset_num_infos = 1002;
};

// tablet writer cancel
message PTabletWriterCancelRequest {
    required PUniqueId id = 1;
    required int64 index_id = 2;
    required int32 sender_id = 3;
};

message PTabletWriterCancelResult {
};

enum PFragmentRequestVersion {
    VERSION_1 = 1;  // only one TExecPlanFragmentParams in request
    VERSION_2 = 2;  // multi TExecPlanFragmentParams in request
    VERSION_3 = 3;  // multi TPipelineFragmentParams in request
    VERSION_4 = 4; // multi TPipelineFragmentParams with optimized common fields in request. Used by 2.1
};

message PExecPlanFragmentRequest {
    optional bytes request = 1;
    optional bool compact = 2;
    optional PFragmentRequestVersion version = 3 [default = VERSION_2];
};

message PExecPlanFragmentStartRequest {
    optional PUniqueId query_id = 1;
}

message PExecPlanFragmentResult {
    required PStatus status = 1;
    // BE receive rpc
    optional int64 received_time = 2;
    // Start executing on bthread
    optional int64 execution_time = 3;
    // Done on bthread
    optional int64 execution_done_time = 4;
};

message PCancelPlanFragmentRequest {
    required PUniqueId finst_id = 1;
    optional PPlanFragmentCancelReason cancel_reason = 2;
    optional PUniqueId query_id = 3;
    optional int32 fragment_id = 4;
    optional PStatus cancel_status = 5;
};

message PCancelPlanFragmentResult {
    required PStatus status = 1;
};

message PFetchDataRequest {
    required PUniqueId finst_id = 1;
    optional bool resp_in_attachment = 2;
};

message PFetchDataResult {
    required PStatus status = 1;
    // valid when status is ok
    optional int64 packet_seq = 2;
    optional bool eos = 3;
    optional PQueryStatistics query_statistics = 4;
    optional bytes row_batch = 5;
    optional bool empty_batch = 6;
};

message PFetchArrowDataRequest {
    optional PUniqueId finst_id = 1;
};

message PFetchArrowDataResult {
    optional PStatus status = 1;
    // valid when status is ok
    optional int64 packet_seq = 2;
    optional bool eos = 3;
    optional PBlock block = 4;
    optional bool empty_batch = 5;
    optional string timezone = 6;
};

message PFetchArrowFlightSchemaRequest {
    optional PUniqueId finst_id = 1;
};

message PFetchArrowFlightSchemaResult {
    optional PStatus status = 1;
    // valid when status is ok
    optional bytes schema = 2;
    optional bytes be_arrow_flight_ip = 3;
    optional int32 be_arrow_flight_port = 4;
};

message KeyTuple {
    repeated string key_column_rep = 1;
}

message UUID {
    required int64 uuid_high = 1;
    required int64 uuid_low = 2;
}

// We use thrift definition for some structure, since TExpr,
// list<Exprs.TExpr>, Descriptors.TDescriptorTable are all thrift format.
// Modify them to protobuf is a redundant work.
message PTabletKeyLookupRequest {
    required int64 tablet_id = 1;
    repeated KeyTuple key_tuples = 2;

    // reusable structures
    // serilized from Descriptors.TDescriptorTable
    optional UUID uuid = 3;
    optional bytes desc_tbl = 4;
    // serilized from TExprList 
    optional bytes output_expr = 5;
    // return binary mysql row format if true
    optional bool is_binary_row = 6;

    // For cloud
    // version to read
    optional int64 version = 7;
    // serilized from TQueryOptions 
    optional bytes query_options = 8;
    optional int64 replica_id = 9;
}

message PTabletBatchKeyLookupRequest {
    repeated PTabletKeyLookupRequest sub_key_lookup_req = 1;
}

message PTabletKeyLookupResponse {
    required PStatus status = 1;
    optional bytes row_batch = 2;
    optional bool empty_batch = 3;
}

message PTabletBatchKeyLookupResponse {
    optional PStatus status = 1;
    repeated PTabletKeyLookupResponse sub_key_lookup_res = 2;
}

//Add message definition to fetch and update cache
enum PCacheStatus {    
    DEFAULT = 0;
    CACHE_OK = 1;
    PARAM_ERROR = 2;
    SIZE_OVER_LIMIT = 3;
    NO_SQL_KEY = 4;
    NO_PARTITION_KEY = 5;
    INVALID_KEY_RANGE = 6;
    DATA_OVERDUE = 7;
    EMPTY_DATA = 8;
    CANCELED = 9;
};

enum CacheType {
    SQL_CACHE = 1;
    PARTITION_CACHE = 2;
};

message PCacheParam {
    required int64 partition_key = 1;
    optional int64 last_version = 2;
    optional int64 last_version_time = 3;
    optional int64 partition_num = 4;
};

message PCacheValue {
    required PCacheParam param = 1;
    required int32 data_size = 2;
    repeated bytes rows = 3;
};

//for update&clear return
message PCacheResponse {
    required PCacheStatus status = 1;
};

message PUpdateCacheRequest{
    required PUniqueId sql_key = 1;
    repeated PCacheValue values = 2;
    optional CacheType cache_type = 3;
};

message PFetchCacheRequest {
    required PUniqueId sql_key = 1;
    repeated PCacheParam params = 2;
};

message PFetchCacheResult {
    required PCacheStatus status = 1;
    repeated PCacheValue values = 2;
    optional int64 all_count = 3 [default = 0];
};

enum PClearType {
    CLEAR_ALL = 0;
    PRUNE_CACHE = 1;
    CLEAR_BEFORE_TIME = 2;
    CLEAR_SQL_KEY = 3;
};

message PClearCacheRequest {
    required PClearType clear_type = 1;
    optional int64 before_time = 2;
    optional PUniqueId sql_key = 3;
};
//End cache proto definition

message PStringPair {
    required string key = 1;
    required string val = 2;
};

message PIntegerPair {
    required int32 key = 1;
    required int64 val = 2;
};

message PKafkaLoadInfo {
    required string brokers = 1;
    required string topic = 2;
    repeated PStringPair properties = 3;
};

message PKafkaMetaProxyRequest {
    optional PKafkaLoadInfo kafka_info = 1;
    // optional for getting offsets for times
    repeated PIntegerPair offset_times = 3;
    // optional for getting latest offsets of partitons
    repeated int32 partition_id_for_latest_offsets = 4;
    // optional for getting real offset for end/beginning flag
    repeated PIntegerPair offset_flags = 5;
};

message PProxyRequest {
    optional PKafkaMetaProxyRequest kafka_meta_request = 1;
    optional int64 timeout_secs = 2;
};

message PKafkaMetaProxyResult {
    repeated int32 partition_ids = 1;
};

message PKafkaPartitionOffsets {
    repeated PIntegerPair offset_times = 1;
};

message PProxyResult {
    required PStatus status = 1;
    optional PKafkaMetaProxyResult kafka_meta_result = 2;
	optional PKafkaPartitionOffsets partition_offsets = 3;
};

message PDataColumn {
    optional string value = 1;
}

message PDataRow {
    repeated PDataColumn col = 1;
}

message PSendDataRequest {
    required PUniqueId fragment_instance_id = 1;
    repeated PDataRow data = 2;
    optional PUniqueId load_id = 3; // load_id == query_id in fragment exec
}

message PSendDataResult {
    required PStatus status = 1;
}

message PCommitRequest {
    required PUniqueId fragment_instance_id = 1;
    optional PUniqueId load_id = 2;
}

message PCommitResult {
    required PStatus status = 1;
}

message PRollbackRequest {
    required PUniqueId fragment_instance_id = 1;
    optional PUniqueId load_id = 2;
}

message PRollbackResult {
    required PStatus status = 1;
}

message PBloomFilter {
     required bool always_true = 2;
     required int32 filter_length = 1;
};

message PColumnValue {
    optional bool boolVal = 1;
    optional int32 intVal = 2;
    optional int64 longVal = 3;
    optional double doubleVal = 4;
    optional bytes stringVal = 5;
    optional int32 precision = 6;
    optional int32 scale = 7;
}

// TODO: CHECK ALL TYPE
enum PColumnType {
    COLUMN_TYPE_BOOL = 0;
    COLUMN_TYPE_INT = 1;
    COLUMN_TYPE_TINY_INT = 2;
    COLUMN_TYPE_SMALL_INT = 3;
    COLUMN_TYPE_BIGINT = 4;
    COLUMN_TYPE_LARGEINT = 5;
    COLUMN_TYPE_VARCHAR = 6;
    COLUMN_TYPE_CHAR = 7;
    COLUMN_TYPE_DATE = 8;
    COLUMN_TYPE_DATETIME = 9;
    COLUMN_TYPE_DOUBLE = 10;
    COLUMN_TYPE_FLOAT = 11;
    COLUMN_TYPE_DECIMAL = 12;
    COLUMN_TYPE_DECIMALV2 = 13;
    COLUMN_TYPE_STRING = 14;
    COLUMN_TYPE_DATEV2 = 15;
    COLUMN_TYPE_DATETIMEV2 = 16;
    COLUMN_TYPE_DECIMAL32 = 17;
    COLUMN_TYPE_DECIMAL64 = 18;
    COLUMN_TYPE_DECIMAL128I = 19;
    COLUMN_TYPE_DECIMAL256 = 20;
    COLUMN_TYPE_IPV4 = 21;
    COLUMN_TYPE_IPV6 = 22;
}

message PMinMaxFilter {
    required PColumnType column_type = 1; // Deprecated
    required PColumnValue min_val = 2;
    required PColumnValue max_val = 3;
};

message PInFilter {
    required PColumnType column_type = 1; // Deprecated
    repeated PColumnValue values = 2;
    optional string ignored_msg = 3;
}

enum PFilterType {
    UNKNOW_FILTER = 0;
    BLOOM_FILTER = 1;
    MINMAX_FILTER = 2;
    IN_FILTER = 3;
    IN_OR_BLOOM_FILTER = 4;
    MIN_FILTER = 5;
    MAX_FILTER = 6;
};

message PSendFilterSizeRequest {
    required int32 filter_id = 1;
    required PUniqueId query_id = 2;
    required PNetworkAddress source_addr = 3;
    required uint64 filter_size = 4;
};

message PSendFilterSizeResponse {
    required PStatus status = 1;
};

message PSyncFilterSizeRequest {
    required int32 filter_id = 1;
    required PUniqueId query_id = 2;
    required uint64 filter_size = 3;
};

message PSyncFilterSizeResponse {
    required PStatus status = 1;
};

message PMergeFilterRequest {
    required int32 filter_id = 1;
    required PUniqueId query_id = 2;
    required PUniqueId fragment_instance_id = 3;
    required PFilterType filter_type = 4;
    optional PMinMaxFilter minmax_filter = 5;
    optional PBloomFilter bloom_filter = 6;
    optional PInFilter in_filter = 7;
    optional bool is_pipeline = 8 [deprecated = true];
    optional bool opt_remote_rf = 9; // Deprecated
    optional PColumnType column_type = 10;
    optional bool contain_null = 11;
    optional bool ignored = 12;
    optional uint64 local_merge_time = 13;
    optional bool disabled = 14;
};

message PMergeFilterResponse {
    required PStatus status = 1;
};

// delete PPublishFilterRequest after upgrade doris 2.1
message PPublishFilterRequest {
    required int32 filter_id = 1;
    required PUniqueId query_id = 2;
    required PUniqueId fragment_instance_id = 3;
    required PFilterType filter_type = 4;
    optional PMinMaxFilter minmax_filter = 5;
    optional PBloomFilter bloom_filter = 6;
    optional PInFilter in_filter = 7;
    optional bool is_pipeline = 8 [deprecated = true];
    optional int64 merge_time = 9;
    optional PColumnType column_type = 10;
    optional bool contain_null = 11;
    optional bool ignored = 12;
    optional bool disabled = 13;
};

message PPublishFilterRequestV2 {
    required int32 filter_id = 1;
    required PUniqueId query_id = 2;
    repeated PUniqueId fragment_instance_ids = 3;
    required PFilterType filter_type = 4;
    optional PMinMaxFilter minmax_filter = 5;
    optional PBloomFilter bloom_filter = 6;
    optional PInFilter in_filter = 7;
    optional bool is_pipeline = 8 [deprecated = true];
    optional int64 merge_time = 9;
    optional bool contain_null = 10;
    optional bool ignored = 11;
    repeated int32 fragment_ids = 12;
    optional uint64 local_merge_time = 13;
    optional bool disabled = 14;
};

message PPublishFilterResponse {
    required PStatus status = 1;
};

message PExprResult {
    required PScalarType type = 1;
    required string content = 2; //maybe need remove after next version of 2.1
    required bool success = 3;
    optional PTypeDesc type_desc = 4;
    optional PValues result_content = 5;
};

message PExprResultMap {
    map<string, PExprResult> map = 1;
};

message PConstantExprRequest {
	optional bytes request = 1;
};

message PConstantExprResult {
    required PStatus status = 1;
    map<string, PExprResultMap> expr_result_map = 2;
};

message PCheckRPCChannelRequest {
    optional bytes data = 1;
    optional uint32 size = 2;
    optional string md5 = 3;
};

message PCheckRPCChannelResponse {
    required PStatus status = 1;
};

message PResetRPCChannelRequest {
    required bool all = 1;
    repeated string endpoints = 2;
};

message PResetRPCChannelResponse {
    required PStatus status = 1;
    repeated string channels = 2;
};

message PEmptyRequest {};

message PTabletWriteSlaveRequest {
    message IndexSize {
        required int64 indexId = 1;
        required int64 size = 2;
        optional string suffix_path = 3;
    };

    message IndexSizeMap{
        repeated IndexSize index_sizes = 1;
    };

    optional RowsetMetaPB rowset_meta = 1;
    optional string rowset_path = 2;
    map<int64, int64> segments_size = 3;
    optional string host = 4;
    optional int32 http_port = 5;
    optional int32 brpc_port = 6;
    optional string token = 7;
    optional int32 node_id = 8;
    map<int64, IndexSizeMap> inverted_indices_size = 9;
};

message PTabletWriteSlaveResult {
    optional PStatus status = 1;
};

message PTabletWriteSlaveDoneRequest {
    optional int64 txn_id = 1;
    optional int64 tablet_id = 2;
    optional int64 node_id = 3;
    optional bool is_succeed = 4 [default = false];
};

message PTabletWriteSlaveDoneResult {
    optional PStatus status = 1;
};

message PFetchTableSchemaRequest {
  optional bytes file_scan_range = 1;
};

message PFetchTableSchemaResult {
  optional PStatus status = 1;
  optional int32 column_nums = 2;
  repeated string column_names = 3;
  repeated PTypeDesc column_types = 4;
}

message POutfileWriteSuccessRequest {
    optional bytes result_file_sink = 1;
}

message POutfileWriteSuccessResult {
    optional PStatus status = 1;
}

message PJdbcTestConnectionRequest {
    optional bytes jdbc_table = 1;
    optional int32 jdbc_table_type = 2;
    optional string query_str = 3;
}

message PJdbcTestConnectionResult {
    optional PStatus status = 1;
}

message PRowLocation {
    optional int64 tablet_id = 1;
    optional string rowset_id = 2;
    optional uint64 segment_id = 3;
    optional uint64 ordinal_id = 4;
}

message PMultiGetRequest {
    repeated PRowLocation row_locs = 1;
    optional PTupleDescriptor desc = 2;
    repeated PSlotDescriptor slots = 3;
    // for compability
    optional int32 be_exec_version = 4;
    optional bool fetch_row_store = 5;
    optional PUniqueId query_id = 6;
    repeated ColumnPB column_desc = 7;
};

message PMultiGetResponse {
    optional PBlock block = 1;
    optional PStatus status = 2;
 
    // more effecient serialization fields for row store
    enum RowFormat {
        JSONB = 0;
    };
    optional RowFormat format = 3;
    repeated bytes binary_row_data = 4;
    // for sorting rows
    repeated PRowLocation row_locs = 5;
};

message PFetchColIdsRequest {
    message PFetchColIdParam {
        required int64 indexId = 1;
        repeated int64 tablet_ids = 2;
    };
    repeated PFetchColIdParam params = 1;
};

message PFetchColIdsResponse {
    message PFetchColIdsResultEntry {
        required int64 index_id = 1;
        map<string, int32> col_name_to_id = 2;
    };
    optional PStatus status = 1;
    repeated PFetchColIdsResultEntry entries = 2;
};

message PGetTabletVersionsRequest {
    required int64 tablet_id = 1;
};

message PGetTabletVersionsResponse {
    message PVersion {
        required int64 first = 1;
        required int64 second = 2;
    };
    required PStatus status = 1;
    repeated PVersion versions = 2;
};

message PGetFileCacheMetaRequest {
    repeated int64 tablet_ids = 1;
}

enum FileCacheType {
    TTL = 0;
    INDEX = 1;
    NORMAL = 2;
}

message FileCacheBlockMeta {
    required int64 tablet_id = 1;
    required string rowset_id = 2;
    required int64 segment_id = 3;
    required string file_name = 4;
    required int64 offset = 5;
    required int64 size = 6;
    required FileCacheType cache_type = 7;
    required int64 expiration_time = 8;
    optional int64 file_size = 9;
}

message PGetFileCacheMetaResponse {
    repeated FileCacheBlockMeta file_cache_block_metas = 1;
}

message PReportStreamLoadStatusRequest {
    optional PUniqueId load_id = 1;
    optional PStatus status = 2;
}

message PReportStreamLoadStatusResponse {
    optional PStatus status = 1;
}

message PGlobRequest {
    optional string pattern = 1;
}

message PGlobResponse {
  message PFileInfo {
    optional string file = 1;
    optional int64 size = 2;
  };
  required PStatus status = 1;
  repeated PFileInfo files = 2;
}

message PGroupCommitInsertRequest {
    optional int64 db_id = 1; // deprecated
    optional int64 table_id = 2; // deprecated
    //  Descriptors.TDescriptorTable
    // optional bytes desc_tbl = 3;
    optional int64 base_schema_version = 4; // deprecated

    // TExecPlanFragmentParams -> TPlanFragment -> PlanNodes.TPlan
    // optional bytes plan_node = 5;
    // TScanRangeParams
    // optional bytes scan_range_params = 6;

    optional PUniqueId load_id = 7;
    repeated PDataRow data = 8;
    // TExecPlanFragmentParams
    optional PExecPlanFragmentRequest exec_plan_fragment_request = 9;
}

message PGroupCommitInsertResponse {
    required PStatus status = 1;
    optional string label = 2;
    optional int64 txn_id = 3;
    optional int64 loaded_rows = 4;
    optional int64 filtered_rows = 5;
    optional string error_url = 6;
}

message POpenLoadStreamRequest {
    optional PUniqueId load_id = 1;
    optional int64 txn_id = 2;
    optional int64 src_id = 3;
    optional POlapTableSchemaParam schema = 4;
    repeated PTabletID tablets = 5;
    optional bool enable_profile = 6 [default = false];
    optional int64 total_streams = 7;
    optional int64 idle_timeout_ms = 8;
}

message PTabletSchemaWithIndex {
    optional int64 index_id = 1;
    optional TabletSchemaPB tablet_schema = 2;
    optional bool enable_unique_key_merge_on_write = 3;
}

message POpenLoadStreamResponse {
    optional PStatus status = 1;
    repeated PTabletSchemaWithIndex tablet_schemas = 2;
}

message PFailedTablet {
    optional int64 id = 1;
    optional PStatus status = 2;
}

message PLoadStreamResponse {
    optional PStatus status = 1;
    repeated int64 success_tablet_ids = 2;
    repeated PFailedTablet failed_tablets = 3;
    optional bytes load_stream_profile = 4;
    repeated PTabletSchemaWithIndex tablet_schemas = 5;
    optional bool eos = 6;
}

message PStreamHeader {
    enum Opcode {
        APPEND_DATA = 1;
        CLOSE_LOAD = 2;
        ADD_SEGMENT = 3;
        GET_SCHEMA = 4;
    }
    optional PUniqueId load_id = 1;
    optional int64 partition_id = 2;
    optional int64 index_id = 3;
    optional int64 tablet_id = 4;
    optional int32 segment_id = 5;
    optional Opcode opcode = 6;
    optional bool segment_eos = 7;
    optional int64 src_id = 8;
    optional SegmentStatisticsPB segment_statistics = 9;
    repeated PTabletID tablets = 10;
    optional TabletSchemaPB flush_schema = 11;
    optional uint64 offset = 12;
    optional FileType file_type = 13;
}

message PGetWalQueueSizeRequest{
    optional int64 table_id = 1;
}

message PGetWalQueueSizeResponse{
    required PStatus status = 1;
    optional int64 size = 2;
}

message PTabletsLocation {
    required string host = 1;
    required int32  brpc_port = 2;
    repeated int64  tablet_id = 3;
}

message PFetchRemoteSchemaRequest {
    repeated PTabletsLocation tablet_location = 1;
    required bool is_coordinator = 2;
}

message PFetchRemoteSchemaResponse {
    optional PStatus status = 1;
    // intermediate merged schema
    optional TabletSchemaPB merged_schema = 2;
}

message PAlterVaultSyncRequest {
}

message PAlterVaultSyncResponse {
}

message PGetBeResourceRequest {
}

message PGlobalResourceUsage {
    optional int64 mem_limit = 1;
    optional int64 mem_usage = 2;
}

message PGetBeResourceResponse {
    optional PStatus status = 1;
    optional PGlobalResourceUsage global_be_resource_usage = 2;
}

service PBackendService {
    rpc transmit_data(PTransmitDataParams) returns (PTransmitDataResult);
    rpc transmit_data_by_http(PEmptyRequest) returns (PTransmitDataResult);
    // If #fragments of a query is < 3, use exec_plan_fragment directly.
    // If #fragments of a query is >=3, use exec_plan_fragment_prepare + exec_plan_fragment_start
    rpc exec_plan_fragment(PExecPlanFragmentRequest) returns (PExecPlanFragmentResult);
    rpc exec_plan_fragment_prepare(PExecPlanFragmentRequest) returns (PExecPlanFragmentResult);
    rpc exec_plan_fragment_start(PExecPlanFragmentStartRequest) returns (PExecPlanFragmentResult);
    rpc cancel_plan_fragment(PCancelPlanFragmentRequest) returns (PCancelPlanFragmentResult);
    rpc fetch_data(PFetchDataRequest) returns (PFetchDataResult);
    rpc fetch_arrow_data(PFetchArrowDataRequest) returns (PFetchArrowDataResult);
    rpc tablet_writer_open(PTabletWriterOpenRequest) returns (PTabletWriterOpenResult);
    rpc open_load_stream(POpenLoadStreamRequest) returns (POpenLoadStreamResponse);
    rpc tablet_writer_add_block(PTabletWriterAddBlockRequest) returns (PTabletWriterAddBlockResult);
    rpc tablet_writer_add_block_by_http(PEmptyRequest) returns (PTabletWriterAddBlockResult);
    rpc tablet_writer_cancel(PTabletWriterCancelRequest) returns (PTabletWriterCancelResult);
    rpc get_info(PProxyRequest) returns (PProxyResult); 
    rpc update_cache(PUpdateCacheRequest) returns (PCacheResponse);
    rpc fetch_cache(PFetchCacheRequest) returns (PFetchCacheResult);
    rpc clear_cache(PClearCacheRequest) returns (PCacheResponse);
    rpc send_data(PSendDataRequest) returns (PSendDataResult);
    rpc commit(PCommitRequest) returns (PCommitResult);
    rpc rollback(PRollbackRequest) returns (PRollbackResult);
    rpc merge_filter(PMergeFilterRequest) returns (PMergeFilterResponse);
    rpc send_filter_size(PSendFilterSizeRequest) returns (PSendFilterSizeResponse);
    rpc sync_filter_size(PSyncFilterSizeRequest) returns (PSyncFilterSizeResponse);
    rpc apply_filter(PPublishFilterRequest) returns (PPublishFilterResponse);
    rpc apply_filterv2(PPublishFilterRequestV2) returns (PPublishFilterResponse);
    rpc fold_constant_expr(PConstantExprRequest) returns (PConstantExprResult);
    rpc transmit_block(PTransmitDataParams) returns (PTransmitDataResult);
    rpc transmit_block_by_http(PEmptyRequest) returns (PTransmitDataResult);
    rpc check_rpc_channel(PCheckRPCChannelRequest) returns (PCheckRPCChannelResponse);
    rpc reset_rpc_channel(PResetRPCChannelRequest) returns (PResetRPCChannelResponse);
    rpc hand_shake(PHandShakeRequest) returns (PHandShakeResponse);
    rpc request_slave_tablet_pull_rowset(PTabletWriteSlaveRequest) returns (PTabletWriteSlaveResult);
    rpc response_slave_tablet_pull_rowset(PTabletWriteSlaveDoneRequest) returns (PTabletWriteSlaveDoneResult);
    rpc outfile_write_success(POutfileWriteSuccessRequest) returns (POutfileWriteSuccessResult);
    rpc fetch_table_schema(PFetchTableSchemaRequest) returns (PFetchTableSchemaResult);
    rpc multiget_data(PMultiGetRequest) returns (PMultiGetResponse);
    rpc get_file_cache_meta_by_tablet_id(PGetFileCacheMetaRequest) returns (PGetFileCacheMetaResponse);
    rpc tablet_fetch_data(PTabletKeyLookupRequest) returns (PTabletKeyLookupResponse);
    rpc tablet_batch_fetch_data(PTabletBatchKeyLookupRequest) returns (PTabletBatchKeyLookupResponse);
    rpc get_column_ids_by_tablet_ids(PFetchColIdsRequest) returns (PFetchColIdsResponse);
    rpc get_tablet_rowset_versions(PGetTabletVersionsRequest) returns (PGetTabletVersionsResponse);
    rpc report_stream_load_status(PReportStreamLoadStatusRequest) returns (PReportStreamLoadStatusResponse);
    rpc glob(PGlobRequest) returns (PGlobResponse);
    rpc group_commit_insert(PGroupCommitInsertRequest) returns (PGroupCommitInsertResponse);
    rpc get_wal_queue_size(PGetWalQueueSizeRequest) returns(PGetWalQueueSizeResponse);
    rpc fetch_arrow_flight_schema(PFetchArrowFlightSchemaRequest) returns (PFetchArrowFlightSchemaResult);
    rpc fetch_remote_tablet_schema(PFetchRemoteSchemaRequest) returns (PFetchRemoteSchemaResponse);
    rpc test_jdbc_connection(PJdbcTestConnectionRequest) returns (PJdbcTestConnectionResult);
    rpc alter_vault_sync(PAlterVaultSyncRequest) returns (PAlterVaultSyncResponse);
    rpc get_be_resource(PGetBeResourceRequest) returns (PGetBeResourceResponse);
};