| /** |
| * Licensed to the Apache Software Foundation (ASF) under one |
| * or more contributor license agreements. See the NOTICE file |
| * distributed with this work for additional information |
| * regarding copyright ownership. The ASF licenses this file |
| * to you under the Apache License, Version 2.0 (the |
| * "License"); you may not use this file except in compliance |
| * with the License. You may obtain a copy of the License at |
| * |
| * http://www.apache.org/licenses/LICENSE-2.0 |
| * |
| * Unless required by applicable law or agreed to in writing, software |
| * distributed under the License is distributed on an "AS IS" BASIS, |
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| * See the License for the specific language governing permissions and |
| * limitations under the License. |
| */ |
| |
| /** |
| * These .proto interfaces are private and stable. |
| * Please see http://wiki.apache.org/hadoop/Compatibility |
| * for what changes are allowed for a *stable* .proto interface. |
| */ |
| |
| // This file contains protocol buffers that are used to transfer data |
| // to and from the datanode, as well as between datanodes. |
| |
| option java_package = "org.apache.hadoop.hdfs.protocol.proto"; |
| option java_outer_classname = "DataTransferProtos"; |
| option java_generate_equals_and_hash = true; |
| package hadoop.hdfs; |
| |
| import "Security.proto"; |
| import "hdfs.proto"; |
| |
| message DataTransferEncryptorMessageProto { |
| enum DataTransferEncryptorStatus { |
| SUCCESS = 0; |
| ERROR_UNKNOWN_KEY = 1; |
| ERROR = 2; |
| } |
| required DataTransferEncryptorStatus status = 1; |
| optional bytes payload = 2; |
| optional string message = 3; |
| repeated CipherOptionProto cipherOption = 4; |
| } |
| |
| message BaseHeaderProto { |
| required ExtendedBlockProto block = 1; |
| optional hadoop.common.TokenProto token = 2; |
| optional DataTransferTraceInfoProto traceInfo = 3; |
| } |
| |
| message DataTransferTraceInfoProto { |
| required uint64 traceId = 1; |
| required uint64 parentId = 2; |
| } |
| |
| message ClientOperationHeaderProto { |
| required BaseHeaderProto baseHeader = 1; |
| required string clientName = 2; |
| } |
| |
| message CachingStrategyProto { |
| optional bool dropBehind = 1; |
| optional int64 readahead = 2; |
| } |
| |
| message OpReadBlockProto { |
| required ClientOperationHeaderProto header = 1; |
| required uint64 offset = 2; |
| required uint64 len = 3; |
| optional bool sendChecksums = 4 [default = true]; |
| optional CachingStrategyProto cachingStrategy = 5; |
| } |
| |
| message ChecksumProto { |
| required ChecksumTypeProto type = 1; |
| required uint32 bytesPerChecksum = 2; |
| } |
| |
| message OpWriteBlockProto { |
| required ClientOperationHeaderProto header = 1; |
| repeated DatanodeInfoProto targets = 2; |
| optional DatanodeInfoProto source = 3; |
| enum BlockConstructionStage { |
| PIPELINE_SETUP_APPEND = 0; |
| // pipeline set up for failed PIPELINE_SETUP_APPEND recovery |
| PIPELINE_SETUP_APPEND_RECOVERY = 1; |
| // data streaming |
| DATA_STREAMING = 2; |
| // pipeline setup for failed data streaming recovery |
| PIPELINE_SETUP_STREAMING_RECOVERY = 3; |
| // close the block and pipeline |
| PIPELINE_CLOSE = 4; |
| // Recover a failed PIPELINE_CLOSE |
| PIPELINE_CLOSE_RECOVERY = 5; |
| // pipeline set up for block creation |
| PIPELINE_SETUP_CREATE = 6; |
| // transfer RBW for adding datanodes |
| TRANSFER_RBW = 7; |
| // transfer Finalized for adding datanodes |
| TRANSFER_FINALIZED = 8; |
| } |
| required BlockConstructionStage stage = 4; |
| required uint32 pipelineSize = 5; |
| required uint64 minBytesRcvd = 6; |
| required uint64 maxBytesRcvd = 7; |
| required uint64 latestGenerationStamp = 8; |
| |
| /** |
| * The requested checksum mechanism for this block write. |
| */ |
| required ChecksumProto requestedChecksum = 9; |
| optional CachingStrategyProto cachingStrategy = 10; |
| optional StorageTypeProto storageType = 11 [default = DISK]; |
| repeated StorageTypeProto targetStorageTypes = 12; |
| |
| /** |
| * Hint to the DataNode that the block can be allocated on transient |
| * storage i.e. memory and written to disk lazily. The DataNode is free |
| * to ignore this hint. |
| */ |
| optional bool allowLazyPersist = 13 [default = false]; |
| //whether to pin the block, so Balancer won't move it. |
| optional bool pinning = 14 [default = false]; |
| repeated bool targetPinnings = 15; |
| optional string storageId = 16; |
| repeated string targetStorageIds = 17; |
| } |
| |
| message OpTransferBlockProto { |
| required ClientOperationHeaderProto header = 1; |
| repeated DatanodeInfoProto targets = 2; |
| repeated StorageTypeProto targetStorageTypes = 3; |
| repeated string targetStorageIds = 4; |
| } |
| |
| message OpReplaceBlockProto { |
| required BaseHeaderProto header = 1; |
| required string delHint = 2; |
| required DatanodeInfoProto source = 3; |
| optional StorageTypeProto storageType = 4 [default = DISK]; |
| optional string storageId = 5; |
| } |
| |
| message OpCopyBlockProto { |
| required BaseHeaderProto header = 1; |
| } |
| |
| message OpBlockChecksumProto { |
| required BaseHeaderProto header = 1; |
| optional BlockChecksumOptionsProto blockChecksumOptions = 2; |
| } |
| |
| message OpBlockGroupChecksumProto { |
| required BaseHeaderProto header = 1; |
| required DatanodeInfosProto datanodes = 2; |
| // each internal block has a block token |
| repeated hadoop.common.TokenProto blockTokens = 3; |
| required ErasureCodingPolicyProto ecPolicy = 4; |
| repeated uint32 blockIndices = 5; |
| required uint64 requestedNumBytes = 6; |
| optional BlockChecksumOptionsProto blockChecksumOptions = 7; |
| } |
| |
| /** |
| * An ID uniquely identifying a shared memory segment. |
| */ |
| message ShortCircuitShmIdProto { |
| required int64 hi = 1; |
| required int64 lo = 2; |
| } |
| |
| /** |
| * An ID uniquely identifying a slot within a shared memory segment. |
| */ |
| message ShortCircuitShmSlotProto { |
| required ShortCircuitShmIdProto shmId = 1; |
| required int32 slotIdx = 2; |
| } |
| |
| message OpRequestShortCircuitAccessProto { |
| required BaseHeaderProto header = 1; |
| |
| /** In order to get short-circuit access to block data, clients must set this |
| * to the highest version of the block data that they can understand. |
| * Currently 1 is the only version, but more versions may exist in the future |
| * if the on-disk format changes. |
| */ |
| required uint32 maxVersion = 2; |
| |
| /** |
| * The shared memory slot to use, if we are using one. |
| */ |
| optional ShortCircuitShmSlotProto slotId = 3; |
| |
| /** |
| * True if the client supports verifying that the file descriptor has been |
| * sent successfully. |
| */ |
| optional bool supportsReceiptVerification = 4 [default = false]; |
| } |
| |
| message ReleaseShortCircuitAccessRequestProto { |
| required ShortCircuitShmSlotProto slotId = 1; |
| optional DataTransferTraceInfoProto traceInfo = 2; |
| } |
| |
| message ReleaseShortCircuitAccessResponseProto { |
| required Status status = 1; |
| optional string error = 2; |
| } |
| |
| message ShortCircuitShmRequestProto { |
| // The name of the client requesting the shared memory segment. This is |
| // purely for logging / debugging purposes. |
| required string clientName = 1; |
| optional DataTransferTraceInfoProto traceInfo = 2; |
| } |
| |
| message ShortCircuitShmResponseProto { |
| required Status status = 1; |
| optional string error = 2; |
| optional ShortCircuitShmIdProto id = 3; |
| } |
| |
| message PacketHeaderProto { |
| // All fields must be fixed-length! |
| required sfixed64 offsetInBlock = 1; |
| required sfixed64 seqno = 2; |
| required bool lastPacketInBlock = 3; |
| required sfixed32 dataLen = 4; |
| optional bool syncBlock = 5 [default = false]; |
| } |
| |
| // Status is a 4-bit enum |
| enum Status { |
| SUCCESS = 0; |
| ERROR = 1; |
| ERROR_CHECKSUM = 2; |
| ERROR_INVALID = 3; |
| ERROR_EXISTS = 4; |
| ERROR_ACCESS_TOKEN = 5; |
| CHECKSUM_OK = 6; |
| ERROR_UNSUPPORTED = 7; |
| OOB_RESTART = 8; // Quick restart |
| OOB_RESERVED1 = 9; // Reserved |
| OOB_RESERVED2 = 10; // Reserved |
| OOB_RESERVED3 = 11; // Reserved |
| IN_PROGRESS = 12; |
| ERROR_BLOCK_PINNED = 13; |
| } |
| |
| enum ShortCircuitFdResponse { |
| DO_NOT_USE_RECEIPT_VERIFICATION = 0; |
| USE_RECEIPT_VERIFICATION = 1; |
| } |
| |
| message PipelineAckProto { |
| required sint64 seqno = 1; |
| repeated Status reply = 2; |
| optional uint64 downstreamAckTimeNanos = 3 [default = 0]; |
| repeated uint32 flag = 4 [packed=true]; |
| } |
| |
| /** |
| * Sent as part of the BlockOpResponseProto |
| * for READ_BLOCK and COPY_BLOCK operations. |
| */ |
| message ReadOpChecksumInfoProto { |
| required ChecksumProto checksum = 1; |
| |
| /** |
| * The offset into the block at which the first packet |
| * will start. This is necessary since reads will align |
| * backwards to a checksum chunk boundary. |
| */ |
| required uint64 chunkOffset = 2; |
| } |
| |
| message BlockOpResponseProto { |
| required Status status = 1; |
| |
| optional string firstBadLink = 2; |
| optional OpBlockChecksumResponseProto checksumResponse = 3; |
| optional ReadOpChecksumInfoProto readOpChecksumInfo = 4; |
| |
| /** explanatory text which may be useful to log on the client side */ |
| optional string message = 5; |
| |
| /** If the server chooses to agree to the request of a client for |
| * short-circuit access, it will send a response message with the relevant |
| * file descriptors attached. |
| * |
| * In the body of the message, this version number will be set to the |
| * specific version number of the block data that the client is about to |
| * read. |
| */ |
| optional uint32 shortCircuitAccessVersion = 6; |
| } |
| |
| /** |
| * Message sent from the client to the DN after reading the entire |
| * read request. |
| */ |
| message ClientReadStatusProto { |
| required Status status = 1; |
| } |
| |
| message DNTransferAckProto { |
| required Status status = 1; |
| } |
| |
| message OpBlockChecksumResponseProto { |
| required uint32 bytesPerCrc = 1; |
| required uint64 crcPerBlock = 2; |
| required bytes blockChecksum = 3; |
| optional ChecksumTypeProto crcType = 4; |
| optional BlockChecksumOptionsProto blockChecksumOptions = 5; |
| } |
| |
| message OpCustomProto { |
| required string customId = 1; |
| } |