gensrc/proto/internal_service.proto

// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements.  See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership.  The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License.  You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied.  See the License for the
// specific language governing permissions and limitations
// under the License.

syntax="proto2";

package doris;
option java_package = "org.apache.doris.proto";

import "data.proto";
import "descriptors.proto";
import "types.proto";
import "olap_file.proto";

option cc_generic_services = true;

// Transmit data when process SQL query
message PTransmitDataParams {
    // non-change member
    required PUniqueId finst_id = 1;
    required int32 node_id = 2;
    // Id of this fragment in its role as a sender.
    required int32 sender_id = 3;
    required int32 be_number = 4;

    // if set to true, indicates that no more row batches will be sent
    // for this dest_node_id
    required bool eos = 5;
    optional PRowBatch row_batch = 6;
    // different per packet
    required int64 packet_seq = 7;
    optional PQueryStatistics query_statistics = 8;

    optional PBlock block = 9;
    // transfer the RowBatch to the Controller Attachment
    optional bool transfer_by_attachment = 10 [default = false];
    optional PUniqueId query_id = 11;
};

message PTransmitDataResult {
    optional PStatus status = 1;
};

message PTabletWithPartition {
    required int64 partition_id = 1;
    required int64 tablet_id = 2;
}

message PTabletInfo {
    required int64 tablet_id = 1;
    required int32 schema_hash = 2;
    // Delta Writer will write data to local disk and then check if there are new raw values not in global dict
    // if appears, then it should add the column name to this vector
    repeated string invalid_dict_cols = 3; 
    // total rows num received by DeltaWriter
    optional int64 received_rows = 4;
}

// open a tablet writer
message PTabletWriterOpenRequest {
    required PUniqueId id = 1;
    required int64 index_id = 2;
    required int64 txn_id = 3;
    required POlapTableSchemaParam schema = 4;
    repeated PTabletWithPartition tablets = 5;
    required int32 num_senders = 6;
    required bool need_gen_rollup = 7;  //Deprecated, not used since alter job v2
    optional int64 load_mem_limit = 8;
    optional int64 load_channel_timeout_s = 9;
    optional bool is_high_priority = 10 [default = false];
    optional string sender_ip = 11 [default = ""];
    optional bool is_vectorized = 12 [default = false];
};

message PTabletWriterOpenResult {
    required PStatus status = 1;
};

// add batch to tablet writer
message PTabletWriterAddBatchRequest {
    required PUniqueId id = 1;
    required int64 index_id = 2;
    required int32 sender_id = 3;

    // If this is the last batch from this sender
    optional bool eos = 4;

    required int64 packet_seq = 5;
    repeated int64 tablet_ids = 6;
    // unset if and only if when eos is true
    optional PRowBatch row_batch = 7;
    // only valid when eos is true
    // valid partition ids that would write in this writer
    repeated int64 partition_ids = 8;
    // the backend which send this request
    optional int64 backend_id = 9 [default = -1];
    // transfer the RowBatch to the Controller Attachment
    optional bool transfer_by_attachment = 10 [default = false];
    optional bool is_high_priority = 11 [default = false];
    optional bool write_single_replica = 12 [default = false];
    map<int64, PSlaveTabletNodes> slave_tablet_nodes = 13;
};

message PTabletWriterAddBlockRequest {
    required PUniqueId id = 1;
    required int64 index_id = 2;
    required int32 sender_id = 3;

    // If this is the last batch from this sender
    optional bool eos = 4;

    optional int64 packet_seq = 5;
    repeated int64 tablet_ids = 6;
    // unset if and only if when eos is true
    optional PBlock block = 7;
    // only valid when eos is true
    // valid partition ids that would write in this writer
    repeated int64 partition_ids = 8;
    // the backend which send this request
    optional int64 backend_id = 9 [default = -1];
    // transfer the vectorized::Block to the Controller Attachment
    optional bool transfer_by_attachment = 10 [default = false];
    optional bool is_high_priority = 11 [default = false];
    optional bool write_single_replica = 12 [default = false];
    map<int64, PSlaveTabletNodes> slave_tablet_nodes = 13;
};

message PSlaveTabletNodes {
    repeated PNodeInfo slave_nodes = 1;
}

message PNodeInfo {
    optional int64 id = 1;
    optional int64 option = 2;
    optional string host = 3;
    // used to transfer data between nodes
    optional int32 async_internal_port = 4;
}

message PSuccessSlaveTabletNodeIds {
    repeated int64 slave_node_ids = 1;
}

message PTabletError {
    optional int64 tablet_id = 1;
    optional string msg = 2;
}

message PTabletWriterAddBatchResult {
    required PStatus status = 1;
    repeated PTabletInfo tablet_vec = 2;
    optional int64 execution_time_us = 3;
    optional int64 wait_lock_time_us = 4;
    optional int64 wait_execution_time_us = 5;
    repeated PTabletError tablet_errors = 6;
    map<int64, PSuccessSlaveTabletNodeIds> success_slave_tablet_node_ids = 7;
};

message PTabletWriterAddBlockResult {
    required PStatus status = 1;
    repeated PTabletInfo tablet_vec = 2;
    optional int64 execution_time_us = 3;
    optional int64 wait_lock_time_us = 4;
    optional int64 wait_execution_time_us = 5;
    repeated PTabletError tablet_errors = 6;
    map<int64, PSuccessSlaveTabletNodeIds> success_slave_tablet_node_ids = 7;
};

// tablet writer cancel
message PTabletWriterCancelRequest {
    required PUniqueId id = 1;
    required int64 index_id = 2;
    required int32 sender_id = 3;
};

message PTabletWriterCancelResult {
};

enum PFragmentRequestVersion {
    VERSION_1 = 1;  // only one TExecPlanFragmentParams in request
    VERSION_2 = 2;  // multi TExecPlanFragmentParams in request
};

message PExecPlanFragmentRequest {
    optional bytes request = 1;
    optional bool compact = 2;
    optional PFragmentRequestVersion version = 3 [default = VERSION_2];
};

message PExecPlanFragmentStartRequest {
    optional PUniqueId query_id = 1;
}

message PExecPlanFragmentResult {
    required PStatus status = 1;
};

message PCancelPlanFragmentRequest {
    required PUniqueId finst_id = 1;
    optional PPlanFragmentCancelReason cancel_reason = 2;
};

message PCancelPlanFragmentResult {
    required PStatus status = 1;
};

message PFetchDataRequest {
    required PUniqueId finst_id = 1;
    optional bool resp_in_attachment = 2;
};

message PFetchDataResult {
    required PStatus status = 1;
    // valid when status is ok
    optional int64 packet_seq = 2;
    optional bool eos = 3;
    optional PQueryStatistics query_statistics = 4;
    optional bytes row_batch = 5;
    optional bool empty_batch = 6;
};

//Add message definition to fetch and update cache
enum PCacheStatus {    
    DEFAULT = 0;
    CACHE_OK = 1;
    PARAM_ERROR = 2;
    SIZE_OVER_LIMIT = 3;
    NO_SQL_KEY = 4;
    NO_PARTITION_KEY = 5;
    INVALID_KEY_RANGE = 6;
    DATA_OVERDUE = 7;
    EMPTY_DATA = 8;
};

enum CacheType {
    SQL_CACHE = 1;
    PARTITION_CACHE = 2;
};

message PCacheParam {
    required int64 partition_key = 1;
    optional int64 last_version = 2;
    optional int64 last_version_time = 3;
};

message PCacheValue {
    required PCacheParam param = 1;
    required int32 data_size = 2;
    repeated bytes rows = 3;
};

//for update&clear return
message PCacheResponse {
    required PCacheStatus status = 1;
};

message PUpdateCacheRequest{
    required PUniqueId sql_key = 1;
    repeated PCacheValue values = 2;
    optional CacheType cache_type = 3;
};

message PFetchCacheRequest {
    required PUniqueId sql_key = 1;
    repeated PCacheParam params = 2;
};

message PFetchCacheResult {
    required PCacheStatus status = 1;
    repeated PCacheValue values = 2;
    optional int64 all_count = 3 [default = 0];
};

enum PClearType {
    CLEAR_ALL = 0;
    PRUNE_CACHE = 1;
    CLEAR_BEFORE_TIME = 2;
    CLEAR_SQL_KEY = 3;
};

message PClearCacheRequest {
    required PClearType clear_type = 1;
    optional int64 before_time = 2;
    optional PUniqueId sql_key = 3;
};
//End cache proto definition

message PStringPair {
    required string key = 1;
    required string val = 2;
};

message PIntegerPair {
    required int32 key = 1;
    required int64 val = 2;
};

message PKafkaLoadInfo {
    required string brokers = 1;
    required string topic = 2;
    repeated PStringPair properties = 3;
};

message PKafkaMetaProxyRequest {
    optional PKafkaLoadInfo kafka_info = 1;
    // optional for getting offsets for times
    repeated PIntegerPair offset_times = 3;
    // optional for getting latest offsets of partitons
    repeated int32 partition_id_for_latest_offsets = 4;
};

message PProxyRequest {
    optional PKafkaMetaProxyRequest kafka_meta_request = 1;
};

message PKafkaMetaProxyResult {
    repeated int32 partition_ids = 1;
};

message PKafkaPartitionOffsets {
    repeated PIntegerPair offset_times = 1;
};

message PProxyResult {
    required PStatus status = 1;
    optional PKafkaMetaProxyResult kafka_meta_result = 2;
	optional PKafkaPartitionOffsets partition_offsets = 3;
};

message PDataColumn {
    optional string value = 1;
}

message PDataRow {
    repeated PDataColumn col = 1;
}

message PSendDataRequest {
    required PUniqueId fragment_instance_id = 1;
    repeated PDataRow data = 2;
    optional PUniqueId load_id = 3; // load_id == query_id in fragment exec
}

message PSendDataResult {
    required PStatus status = 1;
}

message PCommitRequest {
    required PUniqueId fragment_instance_id = 1;
    optional PUniqueId load_id = 2;
}

message PCommitResult {
    required PStatus status = 1;
}

message PRollbackRequest {
    required PUniqueId fragment_instance_id = 1;
    optional PUniqueId load_id = 2;
}

message PRollbackResult {
    required PStatus status = 1;
}

message PBloomFilter {
     required bool always_true = 2;
     required int32 filter_length = 1;
};

message PColumnValue {
    optional bool boolVal = 1;
    optional int32 intVal = 2;
    optional int64 longVal = 3;
    optional double doubleVal = 4;
    optional bytes stringVal = 5;
    optional int32 precision = 6;
    optional int32 scale = 7;
}

// TODO: CHECK ALL TYPE
enum PColumnType {
    COLUMN_TYPE_BOOL = 0;
    COLUMN_TYPE_INT = 1;
    COLUMN_TYPE_TINY_INT = 2;
    COLUMN_TYPE_SMALL_INT = 3;
    COLUMN_TYPE_BIGINT = 4;
    COLUMN_TYPE_LARGEINT = 5;
    COLUMN_TYPE_VARCHAR = 6;
    COLUMN_TYPE_CHAR = 7;
    COLUMN_TYPE_DATE = 8;
    COLUMN_TYPE_DATETIME = 9;
    COLUMN_TYPE_DOUBLE = 10;
    COLUMN_TYPE_FLOAT = 11;
    COLUMN_TYPE_DECIMAL = 12;
    COLUMN_TYPE_DECIMALV2 = 13;
    COLUMN_TYPE_STRING = 14;
    COLUMN_TYPE_DATEV2 = 15;
    COLUMN_TYPE_DATETIMEV2 = 16;
    COLUMN_TYPE_DECIMAL32 = 17;
    COLUMN_TYPE_DECIMAL64 = 18;
    COLUMN_TYPE_DECIMAL128I = 19;
}

message PMinMaxFilter {
    required PColumnType column_type = 1;
    required PColumnValue min_val = 2;
    required PColumnValue max_val = 3;
};

message PInFilter {
    required PColumnType column_type = 1;
    repeated PColumnValue values = 2;
    optional string ignored_msg = 3;
}

enum PFilterType {
    UNKNOW_FILTER = 0;
    BLOOM_FILTER = 1;
    MINMAX_FILTER = 2;
    IN_FILTER = 3;
    IN_OR_BLOOM_FILTER = 4;
};

message PMergeFilterRequest {
    required int32 filter_id = 1;
    required PUniqueId query_id = 2;
    required PUniqueId fragment_id = 3;
    required PFilterType filter_type = 4;
    optional PMinMaxFilter minmax_filter = 5;
    optional PBloomFilter bloom_filter = 6;
    optional PInFilter in_filter = 7;
};

message PMergeFilterResponse {
    required PStatus status = 1;
};

message PPublishFilterRequest {
    required int32 filter_id = 1;
    required PUniqueId query_id = 2;
    required PUniqueId fragment_id = 3;
    required PFilterType filter_type = 4;
    optional PMinMaxFilter minmax_filter = 5;
    optional PBloomFilter bloom_filter = 6;
    optional PInFilter in_filter = 7;
};

message PPublishFilterResponse {
    required PStatus status = 1;
};

message PExprResult {
    required PScalarType type = 1;
    required string content = 2;
    required bool success = 3;
};

message PExprResultMap {
    map<string, PExprResult> map = 1;
};

message PConstantExprRequest {
	optional bytes request = 1;
};

message PConstantExprResult {
    required PStatus status = 1;
    map<string, PExprResultMap> expr_result_map = 2;
};

message PCheckRPCChannelRequest {
    optional bytes data = 1;
    optional uint32 size = 2;
    optional string md5 = 3;
};

message PCheckRPCChannelResponse {
    required PStatus status = 1;
};

message PResetRPCChannelRequest {
    required bool all = 1;
    repeated string endpoints = 2;
};

message PResetRPCChannelResponse {
    required PStatus status = 1;
    repeated string channels = 2;
};

message PEmptyRequest {};

message PTabletWriteSlaveRequest {
    optional RowsetMetaPB rowset_meta = 1;
    optional string rowset_path = 2;
    map<int64, int64> segments_size = 3;
    optional string host = 4;
    optional int32 http_port = 5;
    optional int32 brpc_port = 6;
    optional string token = 7;
    optional int32 node_id = 8;
};

message PTabletWriteSlaveResult {
    optional PStatus status = 1;
};

message PTabletWriteSlaveDoneRequest {
    optional int64 txn_id = 1;
    optional int64 tablet_id = 2;
    optional int64 node_id = 3;
    optional bool is_succeed = 4 [default = false];
};

message PTabletWriteSlaveDoneResult {
    optional PStatus status = 1;
};

message PFetchTableSchemaRequest {
  optional bytes file_scan_range = 1;
};

message PFetchTableSchemaResult {
  optional PStatus status = 1;
  optional int32 column_nums = 2;
  repeated string column_names = 3;
  repeated PTypeDesc column_types = 4;
};

service PBackendService {
    rpc transmit_data(PTransmitDataParams) returns (PTransmitDataResult);
    rpc transmit_data_by_http(PEmptyRequest) returns (PTransmitDataResult);
    // If #fragments of a query is < 3, use exec_plan_fragment directly.
    // If #fragments of a query is >=3, use exec_plan_fragment_prepare + exec_plan_fragment_start
    rpc exec_plan_fragment(PExecPlanFragmentRequest) returns (PExecPlanFragmentResult);
    rpc exec_plan_fragment_prepare(PExecPlanFragmentRequest) returns (PExecPlanFragmentResult);
    rpc exec_plan_fragment_start(PExecPlanFragmentStartRequest) returns (PExecPlanFragmentResult);
    rpc cancel_plan_fragment(PCancelPlanFragmentRequest) returns (PCancelPlanFragmentResult);
    rpc fetch_data(PFetchDataRequest) returns (PFetchDataResult);
    rpc tablet_writer_open(PTabletWriterOpenRequest) returns (PTabletWriterOpenResult);
    rpc tablet_writer_add_batch(PTabletWriterAddBatchRequest) returns (PTabletWriterAddBatchResult);
    rpc tablet_writer_add_batch_by_http(PEmptyRequest) returns (PTabletWriterAddBatchResult);
    rpc tablet_writer_add_block(PTabletWriterAddBlockRequest) returns (PTabletWriterAddBlockResult);
    rpc tablet_writer_add_block_by_http(PEmptyRequest) returns (PTabletWriterAddBlockResult);
    rpc tablet_writer_cancel(PTabletWriterCancelRequest) returns (PTabletWriterCancelResult);
    rpc get_info(PProxyRequest) returns (PProxyResult); 
    rpc update_cache(PUpdateCacheRequest) returns (PCacheResponse);
    rpc fetch_cache(PFetchCacheRequest) returns (PFetchCacheResult);
    rpc clear_cache(PClearCacheRequest) returns (PCacheResponse);
    rpc send_data(PSendDataRequest) returns (PSendDataResult);
    rpc commit(PCommitRequest) returns (PCommitResult);
    rpc rollback(PRollbackRequest) returns (PRollbackResult);
    rpc merge_filter(PMergeFilterRequest) returns (PMergeFilterResponse);
    rpc apply_filter(PPublishFilterRequest) returns (PPublishFilterResponse);
    rpc fold_constant_expr(PConstantExprRequest) returns (PConstantExprResult);
    rpc transmit_block(PTransmitDataParams) returns (PTransmitDataResult);
    rpc transmit_block_by_http(PEmptyRequest) returns (PTransmitDataResult);
    rpc check_rpc_channel(PCheckRPCChannelRequest) returns (PCheckRPCChannelResponse);
    rpc reset_rpc_channel(PResetRPCChannelRequest) returns (PResetRPCChannelResponse);
    rpc hand_shake(PHandShakeRequest) returns (PHandShakeResponse);
    rpc request_slave_tablet_pull_rowset(PTabletWriteSlaveRequest) returns (PTabletWriteSlaveResult);
    rpc response_slave_tablet_pull_rowset(PTabletWriteSlaveDoneRequest) returns (PTabletWriteSlaveDoneResult);
    rpc fetch_table_schema(PFetchTableSchemaRequest) returns (PFetchTableSchemaResult);
};