hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/datatransfer.proto

/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

/**
 * These .proto interfaces are private and stable.
 * Please see http://wiki.apache.org/hadoop/Compatibility
 * for what changes are allowed for a *stable* .proto interface.
 */
syntax="proto2";
// This file contains protocol buffers that are used to transfer data
// to and from the datanode, as well as between datanodes.

option java_package = "org.apache.hadoop.hdfs.protocol.proto";
option java_outer_classname = "DataTransferProtos";
option java_generate_equals_and_hash = true;
package hadoop.hdfs;

import "Security.proto";
import "hdfs.proto";

message DataTransferEncryptorMessageProto {
  enum DataTransferEncryptorStatus {
    SUCCESS = 0;
    ERROR_UNKNOWN_KEY = 1;
    ERROR = 2;
  }
  required DataTransferEncryptorStatus status = 1;
  optional bytes payload = 2;
  optional string message = 3;
  repeated CipherOptionProto cipherOption = 4;
  optional HandshakeSecretProto handshakeSecret = 5;
}

message HandshakeSecretProto {
  required bytes secret = 1;
  required string bpid = 2;
}

message BaseHeaderProto {
  required ExtendedBlockProto block = 1;
  optional hadoop.common.TokenProto token = 2;
  optional DataTransferTraceInfoProto traceInfo = 3;
}

message DataTransferTraceInfoProto {
  required uint64 traceId = 1;
  required uint64 parentId = 2;
}

message ClientOperationHeaderProto {
  required BaseHeaderProto baseHeader = 1;
  required string clientName = 2;
}

message CachingStrategyProto {
  optional bool dropBehind = 1;
  optional int64 readahead = 2;
}

message OpReadBlockProto {
  required ClientOperationHeaderProto header = 1;
  required uint64 offset = 2;
  required uint64 len = 3;
  optional bool sendChecksums = 4 [default = true];
  optional CachingStrategyProto cachingStrategy = 5;
}

message ChecksumProto {
  required ChecksumTypeProto type = 1;
  required uint32 bytesPerChecksum = 2;
}
  
message OpWriteBlockProto {
  required ClientOperationHeaderProto header = 1;
  repeated DatanodeInfoProto targets = 2;
  optional DatanodeInfoProto source = 3;
  enum BlockConstructionStage {
    PIPELINE_SETUP_APPEND = 0;
    // pipeline set up for failed PIPELINE_SETUP_APPEND recovery
    PIPELINE_SETUP_APPEND_RECOVERY = 1;
    // data streaming
    DATA_STREAMING = 2;
    // pipeline setup for failed data streaming recovery
    PIPELINE_SETUP_STREAMING_RECOVERY = 3;
    // close the block and pipeline
    PIPELINE_CLOSE = 4;
    // Recover a failed PIPELINE_CLOSE
    PIPELINE_CLOSE_RECOVERY = 5;
    // pipeline set up for block creation
    PIPELINE_SETUP_CREATE = 6;
    // transfer RBW for adding datanodes
    TRANSFER_RBW = 7;
    // transfer Finalized for adding datanodes
    TRANSFER_FINALIZED = 8;
  }
  required BlockConstructionStage stage = 4;
  required uint32 pipelineSize = 5;
  required uint64 minBytesRcvd = 6;
  required uint64 maxBytesRcvd = 7;
  required uint64 latestGenerationStamp = 8;

  /**
   * The requested checksum mechanism for this block write.
   */
  required ChecksumProto requestedChecksum = 9;
  optional CachingStrategyProto cachingStrategy = 10;
  optional StorageTypeProto storageType = 11 [default = DISK];
  repeated StorageTypeProto targetStorageTypes = 12;

  /**
   * Hint to the DataNode that the block can be allocated on transient
   * storage i.e. memory and written to disk lazily. The DataNode is free
   * to ignore this hint.
   */
  optional bool allowLazyPersist = 13 [default = false];
  //whether to pin the block, so Balancer won't move it.
  optional bool pinning = 14 [default = false];
  repeated bool targetPinnings = 15;
  optional string storageId = 16;
  repeated string targetStorageIds = 17;
}
  
message OpTransferBlockProto {
  required ClientOperationHeaderProto header = 1;
  repeated DatanodeInfoProto targets = 2;
  repeated StorageTypeProto targetStorageTypes = 3;
  repeated string targetStorageIds = 4;
}

message OpReplaceBlockProto {
  required BaseHeaderProto header = 1;
  required string delHint = 2;
  required DatanodeInfoProto source = 3;
  optional StorageTypeProto storageType = 4 [default = DISK];
  optional string storageId = 5;
}

message OpCopyBlockProto {
  required BaseHeaderProto header = 1;
}

message OpBlockChecksumProto {
  required BaseHeaderProto header = 1;
  optional BlockChecksumOptionsProto blockChecksumOptions = 2;
}

message OpBlockGroupChecksumProto {
  required BaseHeaderProto header = 1;
  required DatanodeInfosProto datanodes = 2;
  // each internal block has a block token
  repeated hadoop.common.TokenProto blockTokens = 3;
  required ErasureCodingPolicyProto ecPolicy = 4;
  repeated uint32 blockIndices = 5;
  required uint64 requestedNumBytes = 6;
  optional BlockChecksumOptionsProto blockChecksumOptions = 7;
}

/**
 * An ID uniquely identifying a shared memory segment.
 */
message ShortCircuitShmIdProto { 
  required int64 hi = 1;
  required int64 lo = 2;
}

/**
 * An ID uniquely identifying a slot within a shared memory segment.
 */
message ShortCircuitShmSlotProto {
  required ShortCircuitShmIdProto shmId = 1;
  required int32 slotIdx = 2; 
}

message OpRequestShortCircuitAccessProto { 
  required BaseHeaderProto header = 1;

  /** In order to get short-circuit access to block data, clients must set this
   * to the highest version of the block data that they can understand.
   * Currently 1 is the only version, but more versions may exist in the future
   * if the on-disk format changes.
   */
  required uint32 maxVersion = 2;

  /**
   * The shared memory slot to use, if we are using one.
   */
  optional ShortCircuitShmSlotProto slotId = 3;

  /**
   * True if the client supports verifying that the file descriptor has been
   * sent successfully.
   */
  optional bool supportsReceiptVerification = 4 [default = false];
}

message ReleaseShortCircuitAccessRequestProto {
  required ShortCircuitShmSlotProto slotId = 1;
  optional DataTransferTraceInfoProto traceInfo = 2;
}

message ReleaseShortCircuitAccessResponseProto {
  required Status status = 1;
  optional string error = 2;
}

message ShortCircuitShmRequestProto { 
  // The name of the client requesting the shared memory segment.  This is
  // purely for logging / debugging purposes.
  required string clientName = 1;
  optional DataTransferTraceInfoProto traceInfo = 2;
}

message ShortCircuitShmResponseProto { 
  required Status status = 1;
  optional string error = 2;
  optional ShortCircuitShmIdProto id = 3;
}

message PacketHeaderProto {
  // All fields must be fixed-length!
  required sfixed64 offsetInBlock = 1;
  required sfixed64 seqno = 2;
  required bool lastPacketInBlock = 3;
  required sfixed32 dataLen = 4;
  optional bool syncBlock = 5 [default = false];
}

// Status is a 4-bit enum
enum Status {
  SUCCESS = 0;
  ERROR = 1;
  ERROR_CHECKSUM = 2;
  ERROR_INVALID = 3;
  ERROR_EXISTS = 4;
  ERROR_ACCESS_TOKEN = 5;
  CHECKSUM_OK = 6;
  ERROR_UNSUPPORTED = 7;
  OOB_RESTART = 8;            // Quick restart
  OOB_RESERVED1 = 9;          // Reserved
  OOB_RESERVED2 = 10;         // Reserved
  OOB_RESERVED3 = 11;         // Reserved
  IN_PROGRESS = 12;
  ERROR_BLOCK_PINNED = 13;
}

enum ShortCircuitFdResponse {
  DO_NOT_USE_RECEIPT_VERIFICATION = 0;
  USE_RECEIPT_VERIFICATION = 1;
}

message PipelineAckProto {
  required sint64 seqno = 1;
  repeated Status reply = 2;
  optional uint64 downstreamAckTimeNanos = 3 [default = 0];
  repeated uint32 flag = 4 [packed=true];
}

/**
 * Sent as part of the BlockOpResponseProto
 * for READ_BLOCK and COPY_BLOCK operations.
 */
message ReadOpChecksumInfoProto {
  required ChecksumProto checksum = 1;

  /**
   * The offset into the block at which the first packet
   * will start. This is necessary since reads will align
   * backwards to a checksum chunk boundary.
   */
  required uint64 chunkOffset = 2;
}

message BlockOpResponseProto {
  required Status status = 1;

  optional string firstBadLink = 2;
  optional OpBlockChecksumResponseProto checksumResponse = 3;
  optional ReadOpChecksumInfoProto readOpChecksumInfo = 4;

  /** explanatory text which may be useful to log on the client side */
  optional string message = 5;

  /** If the server chooses to agree to the request of a client for
   * short-circuit access, it will send a response message with the relevant
   * file descriptors attached.
   *
   * In the body of the message, this version number will be set to the
   * specific version number of the block data that the client is about to
   * read.
   */
  optional uint32 shortCircuitAccessVersion = 6;
}

/**
 * Message sent from the client to the DN after reading the entire
 * read request.
 */
message ClientReadStatusProto {
  required Status status = 1;
}

message DNTransferAckProto {
  required Status status = 1;
}

message OpBlockChecksumResponseProto {
  required uint32 bytesPerCrc = 1;
  required uint64 crcPerBlock = 2;
  required bytes blockChecksum = 3;
  optional ChecksumTypeProto crcType = 4;
  optional BlockChecksumOptionsProto blockChecksumOptions = 5;
}

message OpCustomProto {
  required string customId = 1;
}