hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto

/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

/**
 * These .proto interfaces are public and stable.
 * Please see http://wiki.apache.org/hadoop/Compatibility
 * for what changes are allowed for a *stable* .proto interface.
 */

option java_package = "org.apache.hadoop.yarn.proto";
option java_outer_classname = "YarnProtos";
option java_generic_services = true;
option java_generate_equals_and_hash = true;
package hadoop.yarn;

import "Security.proto";

message SerializedExceptionProto {
  optional string message = 1;
  optional string trace = 2;
  optional string class_name = 3;
  optional SerializedExceptionProto cause = 4;
}

message ApplicationIdProto {
  optional int32 id = 1;
  optional int64 cluster_timestamp = 2;
}

message ApplicationAttemptIdProto {
  optional ApplicationIdProto application_id = 1;
  optional int32 attemptId = 2;
}

message ContainerIdProto {
  optional ApplicationIdProto app_id = 1;
  optional ApplicationAttemptIdProto app_attempt_id = 2;
  optional int64 id = 3;
}

message ResourceProto {
  optional int64 memory = 1;
  optional int32 virtual_cores = 2;
}

message ResourceUtilizationProto {
  optional int32 pmem = 1;
  optional int32 vmem = 2;
  optional float cpu = 3;
}

message ResourceOptionProto {
  optional ResourceProto resource = 1;
  optional int32 over_commit_timeout = 2;
}

message NodeResourceMapProto {
  optional NodeIdProto node_id = 1;
  optional ResourceOptionProto resource_option = 2;
}

message PriorityProto {
  optional int32 priority = 1;
}

enum ContainerStateProto {
  C_NEW = 1;
  C_RUNNING = 2;
  C_COMPLETE = 3;
}

message ContainerProto {
  optional ContainerIdProto id = 1;
  optional NodeIdProto nodeId = 2;
  optional string node_http_address = 3;
  optional ResourceProto resource = 4;
  optional PriorityProto priority = 5;
  optional hadoop.common.TokenProto container_token = 6;
  optional int32 version = 9 [default = 0];
}

message ContainerReportProto {
  optional ContainerIdProto container_id = 1;
  optional ResourceProto resource = 2;
  optional NodeIdProto node_id = 3;
  optional PriorityProto priority = 4;
  optional int64 creation_time = 5;
  optional int64 finish_time = 6;
  optional string diagnostics_info = 7 [default = "N/A"];
  optional string log_url = 8;
  optional int32 container_exit_status = 9;
  optional ContainerStateProto container_state = 10;
  optional string node_http_address = 11;
}

enum YarnApplicationStateProto {
  NEW = 1;
  NEW_SAVING = 2;
  SUBMITTED = 3;
  ACCEPTED = 4;
  RUNNING = 5;
  FINISHED = 6;
  FAILED = 7;
  KILLED = 8;
}

enum YarnApplicationAttemptStateProto {
  APP_ATTEMPT_NEW = 1;
  APP_ATTEMPT_SUBMITTED = 2;
  APP_ATTEMPT_SCHEDULED = 3;
  APP_ATTEMPT_ALLOCATED_SAVING = 4;
  APP_ATTEMPT_ALLOCATED = 5;
  APP_ATTEMPT_LAUNCHED = 6;
  APP_ATTEMPT_FAILED = 7;
  APP_ATTEMPT_RUNNING = 8;
  APP_ATTEMPT_FINISHING = 9;
  APP_ATTEMPT_FINISHED = 10;
  APP_ATTEMPT_KILLED = 11;
}

enum FinalApplicationStatusProto {
  APP_UNDEFINED = 0;
  APP_SUCCEEDED = 1;
  APP_FAILED = 2;
  APP_KILLED = 3;
  APP_ENDED = 4;
}

message URLProto {
  optional string scheme = 1;
  optional string host = 2;
  optional int32 port = 3;
  optional string file = 4;
  optional string userInfo = 5;
}

enum LocalResourceVisibilityProto {
  PUBLIC = 1;
  PRIVATE = 2;
  APPLICATION = 3;
}

enum LocalResourceTypeProto {
  ARCHIVE = 1;
  FILE = 2;
  PATTERN = 3;
}

message LocalResourceProto {
  optional URLProto resource = 1;
  optional int64 size = 2;
  optional int64 timestamp = 3;
  optional LocalResourceTypeProto type = 4;
  optional LocalResourceVisibilityProto visibility = 5;
  optional string pattern = 6;
  optional bool should_be_uploaded_to_shared_cache = 7;
}

message ApplicationResourceUsageReportProto {
  optional int32 num_used_containers = 1;
  optional int32 num_reserved_containers = 2;
  optional ResourceProto used_resources = 3;
  optional ResourceProto reserved_resources = 4;
  optional ResourceProto needed_resources = 5;
  optional int64 memory_seconds = 6;
  optional int64 vcore_seconds = 7;
  optional float queue_usage_percentage = 8;
  optional float cluster_usage_percentage = 9;
  optional int64 preempted_memory_seconds = 10;
  optional int64 preempted_vcore_seconds = 11;
}

message ApplicationReportProto {
  optional ApplicationIdProto applicationId = 1;
  optional string user = 2;
  optional string queue = 3;
  optional string name = 4;
  optional string host = 5;
  optional int32 rpc_port = 6;
  optional hadoop.common.TokenProto client_to_am_token = 7;
  optional YarnApplicationStateProto yarn_application_state = 8;
  optional string trackingUrl = 9;
  optional string diagnostics = 10 [default = "N/A"];
  optional int64 startTime = 11;
  optional int64 finishTime = 12;
  optional FinalApplicationStatusProto final_application_status = 13;
  optional ApplicationResourceUsageReportProto app_resource_Usage = 14;
  optional string originalTrackingUrl = 15;
  optional ApplicationAttemptIdProto currentApplicationAttemptId = 16;
  optional float progress = 17;
  optional string applicationType = 18;
  optional hadoop.common.TokenProto am_rm_token = 19;
  repeated string applicationTags = 20;
  optional LogAggregationStatusProto log_aggregation_status = 21;
  optional bool unmanaged_application = 22 [default = false];
  optional PriorityProto priority = 23;
  optional string appNodeLabelExpression = 24;
  optional string amNodeLabelExpression = 25;
}

enum LogAggregationStatusProto {
  LOG_DISABLED = 1;
  LOG_NOT_START = 2;
  LOG_RUNNING = 3;
  LOG_SUCCEEDED = 4;
  LOG_FAILED = 5;
  LOG_TIME_OUT = 6;
  LOG_RUNNING_WITH_FAILURE = 7;
}

message ApplicationAttemptReportProto {
  optional ApplicationAttemptIdProto application_attempt_id = 1;
  optional string host = 2;
  optional int32 rpc_port = 3;
  optional string tracking_url = 4;
  optional string diagnostics = 5 [default = "N/A"];
  optional YarnApplicationAttemptStateProto yarn_application_attempt_state = 6;
  optional ContainerIdProto am_container_id = 7;
  optional string original_tracking_url = 8;
  optional int64 startTime = 9;
  optional int64 finishTime = 10;
}

enum NodeStateProto {
  NS_NEW = 1; 
  NS_RUNNING = 2; 
  NS_UNHEALTHY = 3;
  NS_DECOMMISSIONED = 4; 
  NS_LOST = 5;
  NS_REBOOTED = 6;
  NS_DECOMMISSIONING = 7;
  NS_SHUTDOWN = 8;
}

message NodeIdProto {
  optional string host = 1;
  optional int32 port = 2;
}

message NodeReportProto {
  optional NodeIdProto nodeId = 1;
  optional string httpAddress = 2;
  optional string rackName = 3;
  optional ResourceProto used = 4;
  optional ResourceProto capability = 5;
  optional int32 numContainers = 6;
  optional NodeStateProto node_state = 7;
  optional string health_report = 8;
  optional int64 last_health_report_time = 9;
  repeated string node_labels = 10;
  optional ResourceUtilizationProto containers_utilization = 11;
  optional ResourceUtilizationProto node_utilization = 12;
}

message NodeIdToLabelsInfoProto {
  optional NodeIdProto nodeId = 1;
  repeated NodeLabelProto nodeLabels = 2;
}

message LabelsToNodeIdsProto {
  optional NodeLabelProto nodeLabels = 1;
  repeated NodeIdProto nodeId = 2;
}

message NodeLabelProto {
  optional string name = 1;
  optional bool isExclusive = 2 [default = true]; 
}

enum ContainerTypeProto {
  APPLICATION_MASTER = 1;
  TASK = 2;
}

message AMBlackListingRequestProto {
  optional bool blacklisting_enabled = 1 [default = false];
  optional float blacklisting_failure_threshold = 2;
}
////////////////////////////////////////////////////////////////////////
////// From AM_RM_Protocol /////////////////////////////////////////////
////////////////////////////////////////////////////////////////////////
message ResourceRequestProto {
  optional PriorityProto priority = 1;
  optional string resource_name = 2;
  optional ResourceProto capability = 3;
  optional int32 num_containers = 4;
  optional bool relax_locality = 5 [default = true];
  optional string node_label_expression = 6;
}

enum AMCommandProto {
  AM_RESYNC = 1;
  AM_SHUTDOWN = 2;
}

message PreemptionMessageProto {
  optional StrictPreemptionContractProto strictContract = 1;
  optional PreemptionContractProto contract = 2;
}

message StrictPreemptionContractProto {
  repeated PreemptionContainerProto container = 1;
}

message PreemptionContractProto {
  repeated PreemptionResourceRequestProto resource = 1;
  repeated PreemptionContainerProto container = 2;
}

message PreemptionContainerProto {
  optional ContainerIdProto id = 1;
}

message PreemptionResourceRequestProto {
  optional ResourceRequestProto resource = 1;
}

message ResourceBlacklistRequestProto {
  repeated string blacklist_additions = 1; 
  repeated string blacklist_removals = 2; 
}

////////////////////////////////////////////////////////////////////////
////// From client_RM_Protocol /////////////////////////////////////////
////////////////////////////////////////////////////////////////////////
message ApplicationSubmissionContextProto {
  optional ApplicationIdProto application_id = 1;
  optional string application_name = 2 [default = "N/A"];
  optional string queue = 3 [default = "default"];
  optional PriorityProto priority = 4;
  optional ContainerLaunchContextProto am_container_spec = 5;
  optional bool cancel_tokens_when_complete = 6 [default = true];
  optional bool unmanaged_am = 7 [default = false];
  optional int32 maxAppAttempts = 8 [default = 0];
  optional ResourceProto resource = 9;
  optional string applicationType = 10 [default = "YARN"];
  optional bool keep_containers_across_application_attempts = 11 [default = false];
  repeated string applicationTags = 12;
  optional int64 attempt_failures_validity_interval = 13 [default = -1];
  optional LogAggregationContextProto log_aggregation_context = 14;
  optional ReservationIdProto reservation_id = 15;
  optional string node_label_expression = 16;
  optional ResourceRequestProto am_container_resource_request = 17;
}

message LogAggregationContextProto {
 optional string include_pattern = 1 [default = ".*"];
 optional string exclude_pattern = 2 [default = ""];
 optional string rolled_logs_include_pattern = 3 [default = ""];
 optional string rolled_logs_exclude_pattern = 4 [default = ".*"];
 optional string log_aggregation_policy_class_name = 5;
 optional string log_aggregation_policy_parameters = 6;
}

enum ApplicationAccessTypeProto {
  APPACCESS_VIEW_APP = 1;
  APPACCESS_MODIFY_APP = 2;
}

message ApplicationACLMapProto {
  optional ApplicationAccessTypeProto accessType = 1;
  optional string acl = 2 [default = " "];
}

message YarnClusterMetricsProto {
  optional int32 num_node_managers = 1;
  optional int32 num_decommissioned_nms = 2;
  optional int32 num_active_nms = 3;
  optional int32 num_lost_nms = 4;
  optional int32 num_unhealthy_nms = 5;
  optional int32 num_rebooted_nms = 6;
}

enum QueueStateProto {
  Q_STOPPED = 1;
  Q_RUNNING = 2;
}

message QueueStatisticsProto {
  optional int64 numAppsSubmitted = 1;
  optional int64 numAppsRunning = 2;
  optional int64 numAppsPending = 3;
  optional int64 numAppsCompleted = 4;
  optional int64 numAppsKilled = 5;
  optional int64 numAppsFailed = 6;
  optional int64 numActiveUsers = 7;
  optional int64 availableMemoryMB = 8;
  optional int64 allocatedMemoryMB = 9;
  optional int64 pendingMemoryMB = 10;
  optional int64 reservedMemoryMB = 11;
  optional int64 availableVCores = 12;
  optional int64 allocatedVCores = 13;
  optional int64 pendingVCores = 14;
  optional int64 reservedVCores = 15;
  optional int64 allocatedContainers = 16;
  optional int64 pendingContainers = 17;
  optional int64 reservedContainers = 18;
}

message QueueInfoProto {
  optional string queueName = 1;
  optional float capacity = 2;
  optional float maximumCapacity = 3;
  optional float currentCapacity = 4;
  optional QueueStateProto state = 5;
  repeated QueueInfoProto childQueues = 6;
  repeated ApplicationReportProto applications = 7;
  repeated string accessibleNodeLabels = 8;
  optional string defaultNodeLabelExpression = 9;
  optional QueueStatisticsProto queueStatistics = 10;
  optional bool preemptionDisabled = 11;
}

enum QueueACLProto {
  QACL_SUBMIT_APPLICATIONS = 1;
  QACL_ADMINISTER_QUEUE = 2;
}

message QueueUserACLInfoProto {
  optional string queueName = 1;
  repeated QueueACLProto userAcls = 2;
}

enum SignalContainerCommandProto {
  OUTPUT_THREAD_DUMP = 1;
  GRACEFUL_SHUTDOWN = 2;
  FORCEFUL_SHUTDOWN = 3;
}

////////////////////////////////////////////////////////////////////////
////// From reservation_protocol /////////////////////////////////////
////////////////////////////////////////////////////////////////////////

message ReservationIdProto {
  optional int64 id = 1;
  optional int64 cluster_timestamp = 2;
}

message ReservationRequestProto {
  optional ResourceProto capability = 1;
  optional int32 num_containers = 2 [default = 1];
  optional int32 concurrency = 3 [default = 1];
  optional int64 duration = 4 [default = -1];
}

message ReservationRequestsProto {
  repeated ReservationRequestProto reservation_resources = 1;
  optional ReservationRequestInterpreterProto interpreter = 2 [default = R_ALL];
}

message ReservationDefinitionProto {
  optional ReservationRequestsProto reservation_requests = 1;
  optional int64 arrival = 2;
  optional int64 deadline = 3;
  optional string reservation_name = 4;
}

message ResourceAllocationRequestProto {
  optional int64 start_time = 1;
  optional int64 end_time = 2;
  optional ResourceProto resource = 3;
}

message ReservationAllocationStateProto {
  optional ReservationDefinitionProto reservation_definition = 1;
  repeated ResourceAllocationRequestProto allocation_requests = 2;
  optional int64 start_time = 3;
  optional int64 end_time = 4;
  optional string user = 5;
  optional bool contains_gangs = 6;
  optional int64 acceptance_time = 7;
  optional ReservationIdProto reservation_id = 8;
}

enum ReservationRequestInterpreterProto {
  R_ANY = 0;
  R_ALL = 1;
  R_ORDER = 2;
  R_ORDER_NO_GAP = 3;
 }

////////////////////////////////////////////////////////////////////////
////// From container_manager //////////////////////////////////////////
////////////////////////////////////////////////////////////////////////

message ContainerLaunchContextProto {
  repeated StringLocalResourceMapProto localResources = 1;
  optional bytes tokens = 2;
  repeated StringBytesMapProto service_data = 3;
  repeated StringStringMapProto environment = 4;
  repeated string command = 5;
  repeated ApplicationACLMapProto application_ACLs = 6;
}

message ContainerStatusProto {
  optional ContainerIdProto container_id = 1;
  optional ContainerStateProto state = 2;
  optional string diagnostics = 3 [default = "N/A"];
  optional int32 exit_status = 4 [default = -1000];
  optional ResourceProto capability = 5;
}

enum ContainerExitStatusProto {
  SUCCESS = 0;
  INVALID = -1000;
  ABORTED = -100;
  DISKS_FAILED = -101;
}


////////////////////////////////////////////////////////////////////////
////// From common//////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////////
message StringLocalResourceMapProto {
  optional string key = 1;
  optional LocalResourceProto value = 2;
}

message StringStringMapProto {
  optional string key = 1;
  optional string value = 2;
}

message StringBytesMapProto {
  optional string key = 1;
  optional bytes value = 2;
}

///////////////////////////////////////////////////////////////////////
/////// DEPRECATED ////////////////////////////////////////////////////
message ContainerResourceIncreaseRequestProto {
  optional ContainerIdProto container_id = 1;
  optional ResourceProto capability = 2;
}

message ContainerResourceIncreaseProto {
  optional ContainerIdProto container_id = 1;
  optional ResourceProto capability = 2;
  optional hadoop.common.TokenProto container_token = 3;
}

message ContainerResourceDecreaseProto {
  optional ContainerIdProto container_id = 1;
  optional ResourceProto capability = 2;
}