tajo-core/src/main/proto/ResourceProtos.proto

/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

option java_package = "org.apache.tajo";
option java_outer_classname = "ResourceProtos";
option java_generic_services = false;
option java_generate_equals_and_hash = true;

import "tajo_protos.proto";
import "TajoIdProtos.proto";
import "CatalogProtos.proto";
import "PrimitiveProtos.proto";
import "Plan.proto";
import "errors.proto";
import "stacktrace.proto";

enum ResponseCommand {
  NORMAL = 1;  // ping
  MEMBERSHIP = 2;  // request membership to worker node
  ABORT_QUERY = 3;  //query master failure
  SHUTDOWN = 4;  // black list
}

// TODO add node health information
message NodeStatusProto {
}

enum ResourceType {
  LEAF = 1;
  INTERMEDIATE = 2;
  QUERYMASTER = 3;
}

message AllocationResourceProto {
  required int32 worker_id = 1;
  required NodeResourceProto resource = 2;
}

message ExecutionBlockListProto {
  repeated ExecutionBlockIdProto execution_block_id = 1;
}

message TaskAllocationProto {
  required TaskRequestProto task_request = 1;
  required NodeResourceProto resource = 2;
}

message TaskRequestProto {
  required string query_master_host_and_port = 1;
  required TaskAttemptIdProto id = 2;
  repeated FragmentProto fragments = 3;
  required string output_table = 4;
  required bool clustered_output = 5;
  required LogicalNodeTree plan = 6;
  optional bool inter_query = 7 [default = false];
  repeated FetchProto fetches = 8;
  optional KeyValueSetProto query_context = 9;
  optional DataChannelProto data_channel = 10;
  optional EnforcerProto enforcer = 11;
}

message FetchProto {
  required string host = 1;
  required int32 port = 2;
  required ShuffleType type = 3;
  required ExecutionBlockIdProto execution_block_id = 4;
  required int32 partition_id = 5;
  required string name = 6;
  optional bytes range_start = 7;
  optional bytes range_end = 8;
  optional bool range_last_inclusive = 9;
  optional bool has_next = 10 [default = false];

  // repeated part
  repeated int32 task_id = 11 [packed = true];
  repeated int32 attempt_id = 12 [packed = true];

  optional int64 offset = 13;
  optional int64 length = 14;
}

message TaskStatusProto {
  required TaskAttemptIdProto id = 1;
  required string worker_name = 2;
  required float progress = 3;
  required TaskAttemptState state = 4;
  optional StatSetProto stats = 5;
  optional TableStatsProto input_stats = 6;
  optional TableStatsProto result_stats = 7;
  repeated ShuffleFileOutput shuffle_file_outputs = 8;
}

message TaskCompletionReport {
  required TaskAttemptIdProto id = 1;
  optional StatSetProto stats = 2;
  optional TableStatsProto input_stats = 3;
  optional TableStatsProto result_stats = 4;
  repeated ShuffleFileOutput shuffle_file_outputs = 5;
  repeated PartitionDescProto partitions = 6;
}

message TaskFatalErrorReport {
  required TaskAttemptIdProto id = 1;
  required tajo.error.SerializedException error = 2;
}

message FailureIntermediateProto {
    required int64 page_pos = 1;
    required int32 start_row_num = 2;
    required int32 end_row_num = 3;
}

message IntermediateEntryProto {
  message PageProto {
    required int64 pos = 1;
    required int32 length = 2;
  }
  required ExecutionBlockIdProto eb_id = 1;
  required int32 task_id = 2;
  required int32 attempt_id = 3;
  required int32 part_id = 4;
  required string host = 5;
  required int64 volume = 6;
  repeated PageProto pages = 7;
  repeated FailureIntermediateProto failures = 8;
}

message ExecutionBlockReport {
  required ExecutionBlockIdProto eb_id = 1;
  required bool report_success = 2;
  optional string report_error_message = 3;
  required int32 succeeded_tasks = 4;
  repeated IntermediateEntryProto intermediate_entries = 5;
}

// deprecated
message TaskResponseProto {
  required string id = 1;
  required QueryState status = 2;
}

message StatusReportProto {
  required int64 timestamp = 1;
  required string server_name = 2;
  repeated TaskStatusProto status = 3;
  repeated TaskAttemptIdProto pings = 4;
}

message CommandRequestProto {
  repeated Command command = 1;
}

message CommandResponseProto {
}

message Command {
  required TaskAttemptIdProto id = 1;
  required CommandType type = 2;
}

enum CommandType {
  PREPARE = 0;
  LAUNCH = 1;
  STOP = 2;
  FINALIZE = 3;
}

message ShuffleFileOutput {
  required int32 part_id = 1;
  optional string file_name = 2;
  optional int64 volume = 3;
}

message SessionProto {
  required string session_id = 1;
  required string username = 2;
  required string current_database = 3;
  required int64 last_access_time = 4;
  required KeyValueSetProto variables = 5;
}

message NodeHeartbeatRequest {
  required int32 worker_id = 1;
  optional NodeResourceProto total_resource = 2;
  optional NodeResourceProto available_resource = 3;
  optional int32 running_tasks = 4;
  optional int32 running_query_masters = 5;
  optional WorkerConnectionInfoProto connection_info = 6;
  optional NodeStatusProto status = 7;
}

message NodeHeartbeatResponse {
  required ResponseCommand command = 1 [default = NORMAL];
  optional int32 heart_beat_interval = 2;
  repeated QueryIdProto query_id = 3;
}

// deprecated
message TajoHeartbeatRequest {
  required WorkerConnectionInfoProto connection_info = 1;
  optional QueryIdProto query_id = 2;
  optional QueryState state = 3;
  optional TableDescProto result_desc = 4;
  optional tajo.error.SerializedException error = 5;
  optional float query_progress = 6;
}

// deprecated
message TajoHeartbeatResponse {
  message ResponseCommand {
    required string command = 1;
    repeated string params = 2;
  }
  required BoolProto heartbeat_result = 1;
  optional ResponseCommand response_command = 3;
}

message WorkerConnectionsResponse {
  repeated WorkerConnectionInfoProto worker = 1;
}

message NodeResourceRequest {
  optional string queue = 1;
  required string user_id = 2;
  required ResourceType type = 3;
  required int32 priority = 4;
  required QueryIdProto query_id = 5;
  required int32 num_containers = 6;
  required NodeResourceProto capacity = 7;
  required int32 running_tasks = 8;
  repeated int32 candidate_nodes = 9;
}

message NodeResourceResponse {
  required QueryIdProto query_id = 1;
  repeated AllocationResourceProto resource = 2;
}

message ExecutionBlockContextRequest {
  required ExecutionBlockIdProto execution_block_id = 1;
  required WorkerConnectionInfoProto worker = 2;
}

message ExecutionBlockContextResponse {
  required ExecutionBlockIdProto execution_block_id = 1;
  optional string query_output_path = 2;

  required KeyValueSetProto query_context = 3;
  required string plan_json = 4;
  required ShuffleType shuffle_type = 5;
}

message StopExecutionBlockRequest {
  required ExecutionBlockIdProto execution_block_id = 1;
  optional ExecutionBlockListProto cleanup_list = 2;
}

message BatchAllocationRequest {
  required ExecutionBlockIdProto execution_block_id = 1;
  repeated TaskAllocationProto task_request = 2;
}

message BatchAllocationResponse {
  repeated TaskAllocationProto cancellation_task = 1;
}

message QueryExecutionRequest {
  required QueryIdProto query_id = 1;
  required SessionProto session = 2;
  required KeyValueSetProto query_context = 3;
  required StringProto expr_in_json = 4;
  optional StringProto logical_plan_json = 5;
  required AllocationResourceProto allocation = 6;
}

// Task history
message FetcherHistoryProto {
  required int64 start_time = 1;
  optional int64 finish_time = 2;
  required FetcherState state = 3;
  required int64 file_length = 4;
  required int32 message_received_count = 5;
}

message TaskHistoryProto {
  required TaskAttemptIdProto task_attempt_id = 1;
  required TaskAttemptState state = 2;
  required float progress = 3;
  required int64 start_time = 4;
  required int64 finish_time = 5;
  required TableStatsProto input_stats = 6;
  optional TableStatsProto output_stats = 7;
  optional string output_path = 8;
  optional string working_path = 9;
  optional int32 finished_fetch_count = 10;
  optional int32 total_fetch_count = 11;
  repeated FetcherHistoryProto fetcher_histories = 12;
}