hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto

/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

/**
 * These .proto interfaces are private and stable.
 * Please see https://hadoop.apache.org/docs/current/hadoop-project-dist/hadoop-common/Compatibility.html
 * for what changes are allowed for a *stable* .proto interface.
 */
syntax="proto2";
// This file contains protocol buffers that are used throughout HDFS -- i.e.
// by the client, server, and data transfer protocols.


option java_package = "org.apache.hadoop.hdfs.protocol.proto";
option java_outer_classname = "HdfsProtos";
option java_generate_equals_and_hash = true;
package hadoop.hdfs;

import "Security.proto";
import "acl.proto";

/**
 * Extended block idenfies a block
 */
message ExtendedBlockProto {
  required string poolId = 1;   // Block pool id - globally unique across clusters
  required uint64 blockId = 2;  // the local id within a pool
  required uint64 generationStamp = 3;
  optional uint64 numBytes = 4 [default = 0];  // len does not belong in ebid 
                                               // here for historical reasons
}


/**
* ProvidedStorageLocation will contain the exact location in the provided
  storage. The path, offset and length will result in ranged read. The nonce
  is there to verify that you receive what you expect.
*/

message ProvidedStorageLocationProto {
  required string path = 1;
  required int64 offset = 2;
  required int64 length = 3;
  required bytes nonce = 4;
}

/**
 * Identifies a Datanode
 */
message DatanodeIDProto {
  required string ipAddr = 1;    // IP address
  required string hostName = 2;  // hostname
  required string datanodeUuid = 3;     // UUID assigned to the Datanode. For
                                        // upgraded clusters this is the same
                                        // as the original StorageID of the
                                        // Datanode.
  required uint32 xferPort = 4;  // data streaming port
  required uint32 infoPort = 5;  // datanode http port
  required uint32 ipcPort = 6;   // ipc server port
  optional uint32 infoSecurePort = 7 [default = 0]; // datanode https port
}

/**
 * Datanode local information
 */
message DatanodeLocalInfoProto {
  required string softwareVersion = 1;
  required string configVersion = 2;
  required uint64 uptime = 3;
}

/**
 * Datanode volume information
 */
message DatanodeVolumeInfoProto {
  required string path = 1;
  required StorageTypeProto storageType = 2;
  required uint64 usedSpace = 3;
  required uint64 freeSpace = 4;
  required uint64 reservedSpace = 5;
  required uint64 reservedSpaceForReplicas = 6;
  required uint64 numBlocks = 7;
}

/**
 * DatanodeInfo array
 */
message DatanodeInfosProto {
  repeated DatanodeInfoProto datanodes = 1;
}

/**
 * The status of a Datanode
 */
message DatanodeInfoProto {
  required DatanodeIDProto id = 1;
  optional uint64 capacity = 2 [default = 0];
  optional uint64 dfsUsed = 3 [default = 0];
  optional uint64 remaining = 4 [default = 0];
  optional uint64 blockPoolUsed = 5 [default = 0];
  optional uint64 lastUpdate = 6 [default = 0];
  optional uint32 xceiverCount = 7 [default = 0];
  optional string location = 8;
  optional uint64 nonDfsUsed = 9;
  enum AdminState {
    NORMAL = 0;
    DECOMMISSION_INPROGRESS = 1;
    DECOMMISSIONED = 2;
    ENTERING_MAINTENANCE = 3;
    IN_MAINTENANCE = 4;
  }

  optional AdminState adminState = 10 [default = NORMAL];
  optional uint64 cacheCapacity = 11 [default = 0];
  optional uint64 cacheUsed = 12 [default = 0];
  optional uint64 lastUpdateMonotonic = 13 [default = 0];
  optional string upgradeDomain = 14;
  optional uint64 lastBlockReportTime = 15 [default = 0];
  optional uint64 lastBlockReportMonotonic = 16 [default = 0];
  optional uint32 numBlocks = 17 [default = 0];
}

/**
 * Represents a storage available on the datanode
 */
message DatanodeStorageProto {
  enum StorageState {
    NORMAL = 0;
    READ_ONLY_SHARED = 1;
  }

  required string storageUuid = 1;
  optional StorageState state = 2 [default = NORMAL];
  optional StorageTypeProto storageType = 3 [default = DISK];
}

message StorageReportProto {
  required string storageUuid = 1 [ deprecated = true ];
  optional bool failed = 2 [ default = false ];
  optional uint64 capacity = 3 [ default = 0 ];
  optional uint64 dfsUsed = 4 [ default = 0 ];
  optional uint64 remaining = 5 [ default = 0 ];
  optional uint64 blockPoolUsed = 6 [ default = 0 ];
  optional DatanodeStorageProto storage = 7; // supersedes StorageUuid
  optional uint64 nonDfsUsed = 8;
  optional string mount = 9;
}

/**
 * Summary of a file or directory
 */
message ContentSummaryProto {
  required uint64 length = 1;
  required uint64 fileCount = 2;
  required uint64 directoryCount = 3;
  required uint64 quota = 4;
  required uint64 spaceConsumed = 5;
  required uint64 spaceQuota = 6;
  optional StorageTypeQuotaInfosProto typeQuotaInfos = 7;
  optional uint64 snapshotLength = 8;
  optional uint64 snapshotFileCount = 9;
  optional uint64 snapshotDirectoryCount = 10;
  optional uint64 snapshotSpaceConsumed = 11;
  optional string erasureCodingPolicy = 12;
}

/**
 * Summary of quota usage of a directory
 */
message QuotaUsageProto {
  required uint64 fileAndDirectoryCount = 1;
  required uint64 quota = 2;
  required uint64 spaceConsumed = 3;
  required uint64 spaceQuota = 4;
  optional StorageTypeQuotaInfosProto typeQuotaInfos = 5;
}

/**
 * Storage type quota and usage information of a file or directory
 */
message StorageTypeQuotaInfosProto {
  repeated StorageTypeQuotaInfoProto typeQuotaInfo = 1;
}

message StorageTypeQuotaInfoProto {
  optional StorageTypeProto type = 1 [default = DISK];
  required uint64 quota = 2;
  required uint64 consumed = 3;
}

/**
 * Contains a list of paths corresponding to corrupt files and a cookie
 * used for iterative calls to NameNode.listCorruptFileBlocks.
 *
 */
message CorruptFileBlocksProto {
 repeated string files = 1;
 required string   cookie = 2;
}

/**
 * Types of recognized storage media.
 */
enum StorageTypeProto {
  DISK = 1;
  SSD = 2;
  ARCHIVE = 3;
  RAM_DISK = 4;
  PROVIDED = 5;
  NVDIMM = 6;
}

/**
 * Types of recognized blocks.
 */
enum BlockTypeProto {
  CONTIGUOUS = 0;
  STRIPED = 1;
}

/**
 * A list of storage types. 
 */
message StorageTypesProto {
  repeated StorageTypeProto storageTypes = 1;
}

/**
 * Block replica storage policy.
 */
message BlockStoragePolicyProto {
  required uint32 policyId = 1;
  required string name = 2;
  // a list of storage types for storing the block replicas when creating a
  // block.
  required StorageTypesProto creationPolicy = 3;
  // A list of storage types for creation fallback storage.
  optional StorageTypesProto creationFallbackPolicy = 4;
  optional StorageTypesProto replicationFallbackPolicy = 5;
}


/**
 * A LocatedBlock gives information about a block and its location.
 */ 
message LocatedBlockProto {
  required ExtendedBlockProto b  = 1;
  required uint64 offset = 2;           // offset of first byte of block in the file
  repeated DatanodeInfoProto locs = 3;  // Locations ordered by proximity to client ip
  required bool corrupt = 4;            // true if all replicas of a block are corrupt, else false
                                        // If block has few corrupt replicas, they are filtered and 
                                        // their locations are not part of this object

  required hadoop.common.TokenProto blockToken = 5;
  repeated bool isCached = 6 [packed=true]; // if a location in locs is cached
  repeated StorageTypeProto storageTypes = 7;
  repeated string storageIDs = 8;

  // striped block related fields
  optional bytes blockIndices = 9; // used for striped block to indicate block index for each storage
  repeated hadoop.common.TokenProto blockTokens = 10; // each internal block has a block token
}

message BatchedListingKeyProto {
  required bytes checksum = 1;
  required uint32 pathIndex = 2;
  required bytes startAfter = 3;
}

message DataEncryptionKeyProto {
  required uint32 keyId = 1;
  required string blockPoolId = 2;
  required bytes nonce = 3;
  required bytes encryptionKey = 4;
  required uint64 expiryDate = 5;
  optional string encryptionAlgorithm = 6;
}

/**
 * Cipher suite.
 */
enum CipherSuiteProto {
    UNKNOWN = 1;
    AES_CTR_NOPADDING = 2;
    SM4_CTR_NOPADDING = 3;
}

/**
 * Crypto protocol version used to access encrypted files.
 */
enum CryptoProtocolVersionProto {
    UNKNOWN_PROTOCOL_VERSION = 1;
    ENCRYPTION_ZONES = 2;
}

/**
 * Encryption information for a file.
 */
message FileEncryptionInfoProto {
  required CipherSuiteProto suite = 1;
  required CryptoProtocolVersionProto cryptoProtocolVersion = 2;
  required bytes key = 3;
  required bytes iv = 4;
  required string keyName = 5;
  required string ezKeyVersionName = 6;
}

/**
 * Encryption information for an individual
 * file within an encryption zone
 */
message PerFileEncryptionInfoProto {
  required bytes key = 1;
  required bytes iv = 2;
  required string ezKeyVersionName = 3;
}

/**
 * Encryption information for an encryption
 * zone
 */
message ZoneEncryptionInfoProto {
  required CipherSuiteProto suite = 1;
  required CryptoProtocolVersionProto cryptoProtocolVersion = 2;
  required string keyName = 3;
  optional ReencryptionInfoProto reencryptionProto = 4;
}

/**
 * Re-encryption information for an encryption zone
 */
message ReencryptionInfoProto {
  required string ezKeyVersionName = 1;
  required uint64 submissionTime = 2;
  required bool canceled = 3;
  required int64 numReencrypted = 4;
  required int64 numFailures = 5;
  optional uint64 completionTime = 6;
  optional string lastFile = 7;
}

/**
 * Cipher option
 */
message CipherOptionProto {
  required CipherSuiteProto suite = 1;
  optional bytes inKey = 2;
  optional bytes inIv = 3;
  optional bytes outKey = 4;
  optional bytes outIv = 5;
}

/**
 * A set of file blocks and their locations.
 */
message LocatedBlocksProto {
  required uint64 fileLength = 1;
  repeated LocatedBlockProto blocks = 2;
  required bool underConstruction = 3;
  optional LocatedBlockProto lastBlock = 4;
  required bool isLastBlockComplete = 5;
  optional FileEncryptionInfoProto fileEncryptionInfo = 6;

  // Optional field for erasure coding
  optional ErasureCodingPolicyProto ecPolicy = 7;
}

/**
 * ECSchema options entry
 */
message ECSchemaOptionEntryProto {
  required string key = 1;
  required string value = 2;
}

/**
 * ECSchema for erasurecoding
 */
message ECSchemaProto {
  required string codecName = 1;
  required uint32 dataUnits = 2;
  required uint32 parityUnits = 3;
  repeated ECSchemaOptionEntryProto options = 4;
}

/**
 * EC policy state.
 */
enum ErasureCodingPolicyState {
  DISABLED = 1;
  ENABLED = 2;
  REMOVED = 3;
}

message ErasureCodingPolicyProto {
  optional string name = 1;
  optional ECSchemaProto schema = 2;
  optional uint32 cellSize = 3;
  required uint32 id = 4; // Actually a byte - only 8 bits used
  optional ErasureCodingPolicyState state = 5 [default = ENABLED];
}

message AddErasureCodingPolicyResponseProto {
  required ErasureCodingPolicyProto policy = 1;
  required bool succeed = 2;
  optional string errorMsg = 3;
}

message ECTopologyVerifierResultProto {
  required string resultMessage = 1;
  required bool isSupported = 2;
}

/**
 * Placeholder type for consistent HDFS operations.
 */
message HdfsPathHandleProto {
  optional uint64 inodeId = 1;
  optional uint64 mtime = 2;
  optional string path = 3;
}

/**
 * Status of a file, directory or symlink
 * Optionally includes a file's block locations if requested by client on the rpc call.
 */
message HdfsFileStatusProto {
  enum FileType {
    IS_DIR = 1;
    IS_FILE = 2;
    IS_SYMLINK = 3;
  }
  enum Flags {
    HAS_ACL   = 0x01; // has ACLs
    HAS_CRYPT = 0x02; // encrypted
    HAS_EC    = 0x04; // erasure coded
    SNAPSHOT_ENABLED    = 0x08; // SNAPSHOT ENABLED
  }
  required FileType fileType = 1;
  required bytes path = 2;          // local name of inode encoded java UTF8
  required uint64 length = 3;
  required FsPermissionProto permission = 4;
  required string owner = 5;
  required string group = 6;
  required uint64 modification_time = 7;
  required uint64 access_time = 8;

  // Optional fields for symlink
  optional bytes symlink = 9;             // if symlink, target encoded java UTF8 

  // Optional fields for file
  optional uint32 block_replication = 10 [default = 0]; // only 16bits used
  optional uint64 blocksize = 11 [default = 0];
  optional LocatedBlocksProto locations = 12;  // suppled only if asked by client

  // Optional field for fileId
  optional uint64 fileId = 13 [default = 0]; // default as an invalid id
  optional int32 childrenNum = 14 [default = -1];
  // Optional field for file encryption
  optional FileEncryptionInfoProto fileEncryptionInfo = 15;

  optional uint32 storagePolicy = 16 [default = 0]; // block storage policy id

  // Optional field for erasure coding
  optional ErasureCodingPolicyProto ecPolicy = 17;

  // Set of flags
  optional uint32 flags = 18 [default = 0];
  optional string namespace = 19;
}

/**
 * Checksum algorithms/types used in HDFS
 * Make sure this enum's integer values match enum values' id properties defined
 * in org.apache.hadoop.util.DataChecksum.Type
 */
enum ChecksumTypeProto {
  CHECKSUM_NULL = 0;
  CHECKSUM_CRC32 = 1;
  CHECKSUM_CRC32C = 2;
}

enum BlockChecksumTypeProto {
  MD5CRC = 1;  // BlockChecksum obtained by taking the MD5 digest of chunk CRCs
  COMPOSITE_CRC = 2;  // Chunk-independent CRC, optionally striped
}

/**
 * Algorithms/types denoting how block-level checksums are computed using
 * lower-level chunk checksums/CRCs.
 * These options should be kept in sync with
 * org.apache.hadoop.hdfs.protocol.BlockChecksumOptions.
 */
message BlockChecksumOptionsProto {
  optional BlockChecksumTypeProto blockChecksumType = 1 [default = MD5CRC];

  // Only used if blockChecksumType specifies a striped format, such as
  // COMPOSITE_CRC. If so, then the blockChecksum in the response is expected
  // to be the concatenation of N crcs, where
  // N == ((requestedLength - 1) / stripedLength) + 1
  optional uint64 stripeLength = 2;
}

/**
 * HDFS Server Defaults
 */
message FsServerDefaultsProto {
  required uint64 blockSize = 1;
  required uint32 bytesPerChecksum = 2;
  required uint32 writePacketSize = 3;
  required uint32 replication = 4; // Actually a short - only 16 bits used
  required uint32 fileBufferSize = 5;
  optional bool encryptDataTransfer = 6 [default = false];
  optional uint64 trashInterval = 7 [default = 0];
  optional ChecksumTypeProto checksumType = 8 [default = CHECKSUM_CRC32];
  optional string keyProviderUri = 9;
  optional uint32 policyId = 10 [default = 0];
  optional bool snapshotTrashRootEnabled = 11 [default = false];
}


/**
 * Directory listing
 */
message DirectoryListingProto {
  repeated HdfsFileStatusProto partialListing = 1;
  required uint32 remainingEntries  = 2;
}

message RemoteExceptionProto {
  required string className = 1;
  optional string message = 2;
}

// Directory listing result for a batched listing call.
message BatchedDirectoryListingProto {
  repeated HdfsFileStatusProto partialListing = 1;
  required uint32 parentIdx = 2;
  optional RemoteExceptionProto exception = 3;
}

/**
 * Status of a snapshottable directory: besides the normal information for 
 * a directory status, also include snapshot quota, number of snapshots, and
 * the full path of the parent directory. 
 */
message SnapshottableDirectoryStatusProto {
  required HdfsFileStatusProto dirStatus = 1;

  // Fields specific for snapshottable directory
  required uint32 snapshot_quota = 2;
  required uint32 snapshot_number = 3;
  required bytes parent_fullpath = 4;
}

/**
 * Status of a snapshot directory: besides the normal information for
 * a directory status, also include snapshot ID, and
 * the full path of the parent directory.
 */
message SnapshotStatusProto {
  required HdfsFileStatusProto dirStatus = 1;

  // Fields specific for snapshot directory
  required uint32 snapshotID = 2;
  required bytes parent_fullpath = 3;
  required bool isDeleted = 4;
}


/**
 * Snapshottable directory listing
 */
message SnapshottableDirectoryListingProto {
  repeated SnapshottableDirectoryStatusProto snapshottableDirListing = 1;
}

/**
 * Snapshot listing
 */
message SnapshotListingProto {
  repeated SnapshotStatusProto snapshotListing = 1;
}

/**
 * Snapshot diff report entry
 */
message SnapshotDiffReportEntryProto {
  required bytes fullpath = 1;
  required string modificationLabel = 2;
  optional bytes targetPath = 3;
}

/**
 * Snapshot diff report
 */
message SnapshotDiffReportProto {
  // full path of the directory where snapshots were taken
  required string snapshotRoot = 1;
  required string fromSnapshot = 2;
  required string toSnapshot = 3;
  repeated SnapshotDiffReportEntryProto diffReportEntries = 4;
}

/**
 * Snapshot diff report listing entry
 */
message SnapshotDiffReportListingEntryProto {
  required bytes fullpath = 1;
  required uint64 dirId = 2;
  required bool isReference = 3;
  optional bytes targetPath = 4;
  optional uint64 fileId = 5;
}

message SnapshotDiffReportCursorProto {
  required bytes startPath = 1;
  required int32 index = 2 [default = -1];
}
/**
 * Snapshot diff report listing
 */
message SnapshotDiffReportListingProto {
  // full path of the directory where snapshots were taken
  repeated SnapshotDiffReportListingEntryProto modifiedEntries = 1;
  repeated SnapshotDiffReportListingEntryProto createdEntries = 2;
  repeated SnapshotDiffReportListingEntryProto deletedEntries = 3;
  required bool isFromEarlier = 4;
  optional SnapshotDiffReportCursorProto cursor = 5;
}
/**
 * Block information
 *
 * Please be wary of adding additional fields here, since INodeFiles
 * need to fit in PB's default max message size of 64MB.
 * We restrict the max # of blocks per file
 * (dfs.namenode.fs-limits.max-blocks-per-file), but it's better
 * to avoid changing this.
 */
message BlockProto {
  required uint64 blockId = 1;
  required uint64 genStamp = 2;
  optional uint64 numBytes = 3 [default = 0];
}

/**
 * Information related to a snapshot
 * TODO: add more information
 */
message SnapshotInfoProto {
  required string snapshotName = 1;
  required string snapshotRoot = 2;
  required FsPermissionProto permission = 3;
  required string owner = 4;
  required string group = 5;
  required string createTime = 6;
  // TODO: do we need access time?
}

/**
 * Rolling upgrade status
 */
message RollingUpgradeStatusProto {
  required string blockPoolId = 1;
  optional bool finalized = 2 [default = false];
}


/**
 * A list of storage IDs.
 */
message StorageUuidsProto {
  repeated string storageUuids = 1;
}

/**
 * File access permissions mode.
 */
enum AccessModeProto {
    READ = 1;
    WRITE = 2;
    COPY = 3;
    REPLACE = 4;
}

/**
 * Secret information for the BlockKeyProto. This is not sent on the wire as
 * such but is used to pack a byte array and encrypted and put in
 * BlockKeyProto.bytes
 * When adding further fields, make sure they are optional as they would
 * otherwise not be backwards compatible.
 *
 * Note: As part of the migration from WritableUtils based tokens (aka "legacy")
 * to Protocol Buffers, we use the first byte to determine the type. If the
 * first byte is <=0 then it is a legacy token. This means that when using
 * protobuf tokens, the the first field sent must have a `field_number` less
 * than 16 to make sure that the first byte is positive. Otherwise it could be
 * parsed as a legacy token. See HDFS-11026 for more discussion.
 */
message BlockTokenSecretProto {
  optional uint64 expiryDate = 1;
  optional uint32 keyId = 2;
  optional string userId = 3;
  optional string blockPoolId = 4;
  optional uint64 blockId = 5;
  repeated AccessModeProto modes = 6;
  repeated StorageTypeProto storageTypes = 7;
  repeated string storageIds = 8;
  optional bytes handshakeSecret = 9;
}

/////////////////////////////////////////////////
// Alignment state for namespaces.
/////////////////////////////////////////////////
/**
 * Clients should receive this message in RPC responses and forward it
 * in RPC requests without interpreting it. It should be encoded
 * as an obscure byte array when being sent to clients.
 */
message RouterFederatedStateProto {
  map<string, int64> namespaceStateIds = 1; // Last seen state IDs for multiple namespaces.
}