| /** |
| * Licensed to the Apache Software Foundation (ASF) under one |
| * or more contributor license agreements. See the NOTICE file |
| * distributed with this work for additional information |
| * regarding copyright ownership. The ASF licenses this file |
| * to you under the Apache License, Version 2.0 (the |
| * "License"); you may not use this file except in compliance |
| * with the License. You may obtain a copy of the License at |
| * |
| * http://www.apache.org/licenses/LICENSE-2.0 |
| * |
| * Unless required by applicable law or agreed to in writing, software |
| * distributed under the License is distributed on an "AS IS" BASIS, |
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| * See the License for the specific language governing permissions and |
| * limitations under the License. |
| */ |
| |
| /** |
| * These .proto interfaces are private and stable. |
| * Please see http://wiki.apache.org/hadoop/Compatibility |
| * for what changes are allowed for a *stable* .proto interface. |
| */ |
| syntax="proto2"; |
| // This file contains protocol buffers that are used throughout HDFS -- i.e. |
| // by the client, server, and data transfer protocols. |
| |
| |
| option java_package = "org.apache.hadoop.hdfs.protocol.proto"; |
| option java_outer_classname = "HdfsProtos"; |
| option java_generate_equals_and_hash = true; |
| package hadoop.hdfs; |
| |
| import "Security.proto"; |
| import "acl.proto"; |
| |
| /** |
| * Extended block idenfies a block |
| */ |
| message ExtendedBlockProto { |
| required string poolId = 1; // Block pool id - gloablly unique across clusters |
| required uint64 blockId = 2; // the local id within a pool |
| required uint64 generationStamp = 3; |
| optional uint64 numBytes = 4 [default = 0]; // len does not belong in ebid |
| // here for historical reasons |
| } |
| |
| /** |
| * Identifies a Datanode |
| */ |
| message DatanodeIDProto { |
| required string ipAddr = 1; // IP address |
| required string hostName = 2; // hostname |
| required string datanodeUuid = 3; // UUID assigned to the Datanode. For |
| // upgraded clusters this is the same |
| // as the original StorageID of the |
| // Datanode. |
| required uint32 xferPort = 4; // data streaming port |
| required uint32 infoPort = 5; // datanode http port |
| required uint32 ipcPort = 6; // ipc server port |
| optional uint32 infoSecurePort = 7 [default = 0]; // datanode https port |
| } |
| |
| /** |
| * Datanode local information |
| */ |
| message DatanodeLocalInfoProto { |
| required string softwareVersion = 1; |
| required string configVersion = 2; |
| required uint64 uptime = 3; |
| } |
| |
| /** |
| * Datanode volume information |
| */ |
| message DatanodeVolumeInfoProto { |
| required string path = 1; |
| required StorageTypeProto storageType = 2; |
| required uint64 usedSpace = 3; |
| required uint64 freeSpace = 4; |
| required uint64 reservedSpace = 5; |
| required uint64 reservedSpaceForReplicas = 6; |
| required uint64 numBlocks = 7; |
| } |
| |
| /** |
| * DatanodeInfo array |
| */ |
| message DatanodeInfosProto { |
| repeated DatanodeInfoProto datanodes = 1; |
| } |
| |
| /** |
| * The status of a Datanode |
| */ |
| message DatanodeInfoProto { |
| required DatanodeIDProto id = 1; |
| optional uint64 capacity = 2 [default = 0]; |
| optional uint64 dfsUsed = 3 [default = 0]; |
| optional uint64 remaining = 4 [default = 0]; |
| optional uint64 blockPoolUsed = 5 [default = 0]; |
| optional uint64 lastUpdate = 6 [default = 0]; |
| optional uint32 xceiverCount = 7 [default = 0]; |
| optional string location = 8; |
| optional uint64 nonDfsUsed = 9; |
| enum AdminState { |
| NORMAL = 0; |
| DECOMMISSION_INPROGRESS = 1; |
| DECOMMISSIONED = 2; |
| ENTERING_MAINTENANCE = 3; |
| IN_MAINTENANCE = 4; |
| } |
| |
| optional AdminState adminState = 10 [default = NORMAL]; |
| optional uint64 cacheCapacity = 11 [default = 0]; |
| optional uint64 cacheUsed = 12 [default = 0]; |
| optional uint64 lastUpdateMonotonic = 13 [default = 0]; |
| optional string upgradeDomain = 14; |
| optional uint64 lastBlockReportTime = 15 [default = 0]; |
| optional uint64 lastBlockReportMonotonic = 16 [default = 0]; |
| } |
| |
| /** |
| * Represents a storage available on the datanode |
| */ |
| message DatanodeStorageProto { |
| enum StorageState { |
| NORMAL = 0; |
| READ_ONLY_SHARED = 1; |
| } |
| |
| required string storageUuid = 1; |
| optional StorageState state = 2 [default = NORMAL]; |
| optional StorageTypeProto storageType = 3 [default = DISK]; |
| } |
| |
| message StorageReportProto { |
| required string storageUuid = 1 [ deprecated = true ]; |
| optional bool failed = 2 [ default = false ]; |
| optional uint64 capacity = 3 [ default = 0 ]; |
| optional uint64 dfsUsed = 4 [ default = 0 ]; |
| optional uint64 remaining = 5 [ default = 0 ]; |
| optional uint64 blockPoolUsed = 6 [ default = 0 ]; |
| optional DatanodeStorageProto storage = 7; // supersedes StorageUuid |
| optional uint64 nonDfsUsed = 8; |
| } |
| |
| /** |
| * Summary of a file or directory |
| */ |
| message ContentSummaryProto { |
| required uint64 length = 1; |
| required uint64 fileCount = 2; |
| required uint64 directoryCount = 3; |
| required uint64 quota = 4; |
| required uint64 spaceConsumed = 5; |
| required uint64 spaceQuota = 6; |
| optional StorageTypeQuotaInfosProto typeQuotaInfos = 7; |
| optional uint64 snapshotLength = 8; |
| optional uint64 snapshotFileCount = 9; |
| optional uint64 snapshotDirectoryCount = 10; |
| optional uint64 snapshotSpaceConsumed = 11; |
| optional string erasureCodingPolicy = 12; |
| } |
| |
| /** |
| * Summary of quota usage of a directory |
| */ |
| message QuotaUsageProto { |
| required uint64 fileAndDirectoryCount = 1; |
| required uint64 quota = 2; |
| required uint64 spaceConsumed = 3; |
| required uint64 spaceQuota = 4; |
| optional StorageTypeQuotaInfosProto typeQuotaInfos = 5; |
| } |
| |
| /** |
| * Storage type quota and usage information of a file or directory |
| */ |
| message StorageTypeQuotaInfosProto { |
| repeated StorageTypeQuotaInfoProto typeQuotaInfo = 1; |
| } |
| |
| message StorageTypeQuotaInfoProto { |
| optional StorageTypeProto type = 1 [default = DISK]; |
| required uint64 quota = 2; |
| required uint64 consumed = 3; |
| } |
| |
| /** |
| * Contains a list of paths corresponding to corrupt files and a cookie |
| * used for iterative calls to NameNode.listCorruptFileBlocks. |
| * |
| */ |
| message CorruptFileBlocksProto { |
| repeated string files = 1; |
| required string cookie = 2; |
| } |
| |
| /** |
| * Types of recognized storage media. |
| */ |
| enum StorageTypeProto { |
| DISK = 1; |
| SSD = 2; |
| ARCHIVE = 3; |
| RAM_DISK = 4; |
| } |
| |
| /** |
| * Types of recognized blocks. |
| */ |
| enum BlockTypeProto { |
| CONTIGUOUS = 0; |
| STRIPED = 1; |
| } |
| |
| /** |
| * A list of storage types. |
| */ |
| message StorageTypesProto { |
| repeated StorageTypeProto storageTypes = 1; |
| } |
| |
| /** |
| * Block replica storage policy. |
| */ |
| message BlockStoragePolicyProto { |
| required uint32 policyId = 1; |
| required string name = 2; |
| // a list of storage types for storing the block replicas when creating a |
| // block. |
| required StorageTypesProto creationPolicy = 3; |
| // A list of storage types for creation fallback storage. |
| optional StorageTypesProto creationFallbackPolicy = 4; |
| optional StorageTypesProto replicationFallbackPolicy = 5; |
| } |
| |
| |
| /** |
| * A LocatedBlock gives information about a block and its location. |
| */ |
| message LocatedBlockProto { |
| required ExtendedBlockProto b = 1; |
| required uint64 offset = 2; // offset of first byte of block in the file |
| repeated DatanodeInfoProto locs = 3; // Locations ordered by proximity to client ip |
| required bool corrupt = 4; // true if all replicas of a block are corrupt, else false |
| // If block has few corrupt replicas, they are filtered and |
| // their locations are not part of this object |
| |
| required hadoop.common.TokenProto blockToken = 5; |
| repeated bool isCached = 6 [packed=true]; // if a location in locs is cached |
| repeated StorageTypeProto storageTypes = 7; |
| repeated string storageIDs = 8; |
| |
| // striped block related fields |
| optional bytes blockIndices = 9; // used for striped block to indicate block index for each storage |
| repeated hadoop.common.TokenProto blockTokens = 10; // each internal block has a block token |
| } |
| |
| message DataEncryptionKeyProto { |
| required uint32 keyId = 1; |
| required string blockPoolId = 2; |
| required bytes nonce = 3; |
| required bytes encryptionKey = 4; |
| required uint64 expiryDate = 5; |
| optional string encryptionAlgorithm = 6; |
| } |
| |
| /** |
| * Cipher suite. |
| */ |
| enum CipherSuiteProto { |
| UNKNOWN = 1; |
| AES_CTR_NOPADDING = 2; |
| } |
| |
| /** |
| * Crypto protocol version used to access encrypted files. |
| */ |
| enum CryptoProtocolVersionProto { |
| UNKNOWN_PROTOCOL_VERSION = 1; |
| ENCRYPTION_ZONES = 2; |
| } |
| |
| /** |
| * Encryption information for a file. |
| */ |
| message FileEncryptionInfoProto { |
| required CipherSuiteProto suite = 1; |
| required CryptoProtocolVersionProto cryptoProtocolVersion = 2; |
| required bytes key = 3; |
| required bytes iv = 4; |
| required string keyName = 5; |
| required string ezKeyVersionName = 6; |
| } |
| |
| /** |
| * Encryption information for an individual |
| * file within an encryption zone |
| */ |
| message PerFileEncryptionInfoProto { |
| required bytes key = 1; |
| required bytes iv = 2; |
| required string ezKeyVersionName = 3; |
| } |
| |
| /** |
| * Encryption information for an encryption |
| * zone |
| */ |
| message ZoneEncryptionInfoProto { |
| required CipherSuiteProto suite = 1; |
| required CryptoProtocolVersionProto cryptoProtocolVersion = 2; |
| required string keyName = 3; |
| optional ReencryptionInfoProto reencryptionProto = 4; |
| } |
| |
| /** |
| * Re-encryption information for an encryption zone |
| */ |
| message ReencryptionInfoProto { |
| required string ezKeyVersionName = 1; |
| required uint64 submissionTime = 2; |
| required bool canceled = 3; |
| required int64 numReencrypted = 4; |
| required int64 numFailures = 5; |
| optional uint64 completionTime = 6; |
| optional string lastFile = 7; |
| } |
| |
| /** |
| * Cipher option |
| */ |
| message CipherOptionProto { |
| required CipherSuiteProto suite = 1; |
| optional bytes inKey = 2; |
| optional bytes inIv = 3; |
| optional bytes outKey = 4; |
| optional bytes outIv = 5; |
| } |
| |
| /** |
| * A set of file blocks and their locations. |
| */ |
| message LocatedBlocksProto { |
| required uint64 fileLength = 1; |
| repeated LocatedBlockProto blocks = 2; |
| required bool underConstruction = 3; |
| optional LocatedBlockProto lastBlock = 4; |
| required bool isLastBlockComplete = 5; |
| optional FileEncryptionInfoProto fileEncryptionInfo = 6; |
| |
| // Optional field for erasure coding |
| optional ErasureCodingPolicyProto ecPolicy = 7; |
| } |
| |
| /** |
| * ECSchema options entry |
| */ |
| message ECSchemaOptionEntryProto { |
| required string key = 1; |
| required string value = 2; |
| } |
| |
| /** |
| * ECSchema for erasurecoding |
| */ |
| message ECSchemaProto { |
| required string codecName = 1; |
| required uint32 dataUnits = 2; |
| required uint32 parityUnits = 3; |
| repeated ECSchemaOptionEntryProto options = 4; |
| } |
| |
| /** |
| * EC policy state. |
| */ |
| enum ErasureCodingPolicyState { |
| DISABLED = 1; |
| ENABLED = 2; |
| REMOVED = 3; |
| } |
| |
| message ErasureCodingPolicyProto { |
| optional string name = 1; |
| optional ECSchemaProto schema = 2; |
| optional uint32 cellSize = 3; |
| required uint32 id = 4; // Actually a byte - only 8 bits used |
| optional ErasureCodingPolicyState state = 5 [default = ENABLED]; |
| } |
| |
| message AddErasureCodingPolicyResponseProto { |
| required ErasureCodingPolicyProto policy = 1; |
| required bool succeed = 2; |
| optional string errorMsg = 3; |
| } |
| |
| /** |
| * Status of a file, directory or symlink |
| * Optionally includes a file's block locations if requested by client on the rpc call. |
| */ |
| message HdfsFileStatusProto { |
| enum FileType { |
| IS_DIR = 1; |
| IS_FILE = 2; |
| IS_SYMLINK = 3; |
| } |
| enum Flags { |
| HAS_ACL = 0x01; // has ACLs |
| HAS_CRYPT = 0x02; // encrypted |
| HAS_EC = 0x04; // erasure coded |
| SNAPSHOT_ENABLED = 0x08; // SNAPSHOT ENABLED |
| } |
| required FileType fileType = 1; |
| required bytes path = 2; // local name of inode encoded java UTF8 |
| required uint64 length = 3; |
| required FsPermissionProto permission = 4; |
| required string owner = 5; |
| required string group = 6; |
| required uint64 modification_time = 7; |
| required uint64 access_time = 8; |
| |
| // Optional fields for symlink |
| optional bytes symlink = 9; // if symlink, target encoded java UTF8 |
| |
| // Optional fields for file |
| optional uint32 block_replication = 10 [default = 0]; // only 16bits used |
| optional uint64 blocksize = 11 [default = 0]; |
| optional LocatedBlocksProto locations = 12; // suppled only if asked by client |
| |
| // Optional field for fileId |
| optional uint64 fileId = 13 [default = 0]; // default as an invalid id |
| optional int32 childrenNum = 14 [default = -1]; |
| // Optional field for file encryption |
| optional FileEncryptionInfoProto fileEncryptionInfo = 15; |
| |
| optional uint32 storagePolicy = 16 [default = 0]; // block storage policy id |
| |
| // Optional field for erasure coding |
| optional ErasureCodingPolicyProto ecPolicy = 17; |
| |
| // Set of flags |
| optional uint32 flags = 18 [default = 0]; |
| } |
| |
| /** |
| * Checksum algorithms/types used in HDFS |
| * Make sure this enum's integer values match enum values' id properties defined |
| * in org.apache.hadoop.util.DataChecksum.Type |
| */ |
| enum ChecksumTypeProto { |
| CHECKSUM_NULL = 0; |
| CHECKSUM_CRC32 = 1; |
| CHECKSUM_CRC32C = 2; |
| } |
| |
| /** |
| * HDFS Server Defaults |
| */ |
| message FsServerDefaultsProto { |
| required uint64 blockSize = 1; |
| required uint32 bytesPerChecksum = 2; |
| required uint32 writePacketSize = 3; |
| required uint32 replication = 4; // Actually a short - only 16 bits used |
| required uint32 fileBufferSize = 5; |
| optional bool encryptDataTransfer = 6 [default = false]; |
| optional uint64 trashInterval = 7 [default = 0]; |
| optional ChecksumTypeProto checksumType = 8 [default = CHECKSUM_CRC32]; |
| optional string keyProviderUri = 9; |
| optional uint32 policyId = 10 [default = 0]; |
| } |
| |
| |
| /** |
| * Directory listing |
| */ |
| message DirectoryListingProto { |
| repeated HdfsFileStatusProto partialListing = 1; |
| required uint32 remainingEntries = 2; |
| } |
| |
| /** |
| * Status of a snapshottable directory: besides the normal information for |
| * a directory status, also include snapshot quota, number of snapshots, and |
| * the full path of the parent directory. |
| */ |
| message SnapshottableDirectoryStatusProto { |
| required HdfsFileStatusProto dirStatus = 1; |
| |
| // Fields specific for snapshottable directory |
| required uint32 snapshot_quota = 2; |
| required uint32 snapshot_number = 3; |
| required bytes parent_fullpath = 4; |
| } |
| |
| /** |
| * Snapshottable directory listing |
| */ |
| message SnapshottableDirectoryListingProto { |
| repeated SnapshottableDirectoryStatusProto snapshottableDirListing = 1; |
| } |
| |
| /** |
| * Snapshot diff report entry |
| */ |
| message SnapshotDiffReportEntryProto { |
| required bytes fullpath = 1; |
| required string modificationLabel = 2; |
| optional bytes targetPath = 3; |
| } |
| |
| /** |
| * Snapshot diff report |
| */ |
| message SnapshotDiffReportProto { |
| // full path of the directory where snapshots were taken |
| required string snapshotRoot = 1; |
| required string fromSnapshot = 2; |
| required string toSnapshot = 3; |
| repeated SnapshotDiffReportEntryProto diffReportEntries = 4; |
| } |
| |
| /** |
| * Block information |
| * |
| * Please be wary of adding additional fields here, since INodeFiles |
| * need to fit in PB's default max message size of 64MB. |
| * We restrict the max # of blocks per file |
| * (dfs.namenode.fs-limits.max-blocks-per-file), but it's better |
| * to avoid changing this. |
| */ |
| message BlockProto { |
| required uint64 blockId = 1; |
| required uint64 genStamp = 2; |
| optional uint64 numBytes = 3 [default = 0]; |
| } |
| |
| /** |
| * Information related to a snapshot |
| * TODO: add more information |
| */ |
| message SnapshotInfoProto { |
| required string snapshotName = 1; |
| required string snapshotRoot = 2; |
| required FsPermissionProto permission = 3; |
| required string owner = 4; |
| required string group = 5; |
| required string createTime = 6; |
| // TODO: do we need access time? |
| } |
| |
| /** |
| * Rolling upgrade status |
| */ |
| message RollingUpgradeStatusProto { |
| required string blockPoolId = 1; |
| optional bool finalized = 2 [default = false]; |
| } |
| |
| |
| /** |
| * A list of storage IDs. |
| */ |
| message StorageUuidsProto { |
| repeated string storageUuids = 1; |
| } |
| |
| /** |
| * File access permissions mode. |
| */ |
| enum AccessModeProto { |
| READ = 1; |
| WRITE = 2; |
| COPY = 3; |
| REPLACE = 4; |
| } |
| |
| /** |
| * Secret information for the BlockKeyProto. This is not sent on the wire as |
| * such but is used to pack a byte array and encrypted and put in |
| * BlockKeyProto.bytes |
| * When adding further fields, make sure they are optional as they would |
| * otherwise not be backwards compatible. |
| * |
| * Note: As part of the migration from WritableUtils based tokens (aka "legacy") |
| * to Protocol Buffers, we use the first byte to determine the type. If the |
| * first byte is <=0 then it is a legacy token. This means that when using |
| * protobuf tokens, the the first field sent must have a `field_number` less |
| * than 16 to make sure that the first byte is positive. Otherwise it could be |
| * parsed as a legacy token. See HDFS-11026 for more discussion. |
| */ |
| message BlockTokenSecretProto { |
| optional uint64 expiryDate = 1; |
| optional uint32 keyId = 2; |
| optional string userId = 3; |
| optional string blockPoolId = 4; |
| optional uint64 blockId = 5; |
| repeated AccessModeProto modes = 6; |
| repeated StorageTypeProto storageTypes = 7; |
| repeated string storageIds = 8; |
| } |