HDDS-3631. KeyInfo related changes to support fileHandle. (#955)
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyInfo.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyInfo.java
index ee71fb2..0178fd3 100644
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyInfo.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyInfo.java
@@ -30,6 +30,7 @@
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.ozone.OzoneAcl;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.KeyInfo;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.FileHandle;
import org.apache.hadoop.ozone.protocolPB.OMPBHelper;
import org.apache.hadoop.util.Time;
@@ -52,6 +53,7 @@
private HddsProtos.ReplicationType type;
private HddsProtos.ReplicationFactor factor;
private FileEncryptionInfo encInfo;
+ private long fileHandleInfo;
/**
* ACL Information.
@@ -66,7 +68,7 @@
HddsProtos.ReplicationFactor factor,
Map<String, String> metadata,
FileEncryptionInfo encInfo, List<OzoneAcl> acls,
- long objectID, long updateID) {
+ long objectID, long updateID, long fileHandleInfo) {
this.volumeName = volumeName;
this.bucketName = bucketName;
this.keyName = keyName;
@@ -92,6 +94,11 @@
this.acls = acls;
this.objectID = objectID;
this.updateID = updateID;
+ if (fileHandleInfo != 0) {
+ this.fileHandleInfo = fileHandleInfo;
+ } else {
+ this.fileHandleInfo = objectID;
+ }
}
public String getVolumeName() {
@@ -139,6 +146,14 @@
this.modificationTime = Time.monotonicNow();
}
+ public long getFileHandleInfo() {
+ return fileHandleInfo;
+ }
+
+ public void setFileHandleInfo(long fileHandleInfo) {
+ this.fileHandleInfo = fileHandleInfo;
+ }
+
/**
* updates the length of the each block in the list given.
* This will be called when the key is being committed to OzoneManager.
@@ -271,6 +286,7 @@
private List<OzoneAcl> acls;
private long objectID;
private long updateID;
+ private long fileHandleInfo;
public Builder() {
this.metadata = new HashMap<>();
@@ -368,6 +384,11 @@
return this;
}
+ public Builder setFileHandleInfo(long fh) {
+ this.fileHandleInfo = fh;
+ return this;
+ }
+
public Builder setUpdateID(long id) {
this.updateID = id;
return this;
@@ -377,13 +398,15 @@
return new OmKeyInfo(
volumeName, bucketName, keyName, omKeyLocationInfoGroups,
dataSize, creationTime, modificationTime, type, factor, metadata,
- encInfo, acls, objectID, updateID);
+ encInfo, acls, objectID, updateID, fileHandleInfo);
}
}
public KeyInfo getProtobuf() {
long latestVersion = keyLocationVersions.size() == 0 ? -1 :
keyLocationVersions.get(keyLocationVersions.size() - 1).getVersion();
+ FileHandle.Builder fb = FileHandle.newBuilder()
+ .setFhObjectID(fileHandleInfo);
KeyInfo.Builder kb = KeyInfo.newBuilder()
.setVolumeName(volumeName)
.setBucketName(bucketName)
@@ -400,7 +423,8 @@
.addAllMetadata(KeyValueUtil.toProtobuf(metadata))
.addAllAcls(OzoneAclUtil.toProtobuf(acls))
.setObjectID(objectID)
- .setUpdateID(updateID);
+ .setUpdateID(updateID)
+ .setFileHandleInfo(fb.build());
if (encInfo != null) {
kb.setFileEncryptionInfo(OMPBHelper.convert(encInfo));
}
@@ -433,6 +457,12 @@
if (keyInfo.hasUpdateID()) {
builder.setUpdateID(keyInfo.getUpdateID());
}
+ if (keyInfo.hasFileHandleInfo()) {
+ FileHandle fh = keyInfo.getFileHandleInfo();
+ if (fh.hasFhObjectID()) {
+ builder.setFileHandleInfo(fh.getFhObjectID());
+ }
+ }
return builder.build();
}
@@ -471,7 +501,9 @@
Objects.equals(metadata, omKeyInfo.metadata) &&
Objects.equals(acls, omKeyInfo.acls) &&
objectID == omKeyInfo.objectID &&
- updateID == omKeyInfo.updateID;
+ updateID == omKeyInfo.updateID &&
+ (fileHandleInfo != 0 ?
+ fileHandleInfo == omKeyInfo.fileHandleInfo : true);
}
@Override
@@ -493,8 +525,9 @@
.setReplicationType(type)
.setReplicationFactor(factor)
.setFileEncryptionInfo(encInfo)
- .setObjectID(objectID).setUpdateID(updateID);
-
+ .setObjectID(objectID)
+ .setUpdateID(updateID)
+ .setFileHandleInfo(fileHandleInfo);
keyLocationVersions.forEach(keyLocationVersion -> {
List<OmKeyLocationInfo> keyLocationInfos = new ArrayList<>();
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmMultipartKeyInfo.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmMultipartKeyInfo.java
index df8751c..79b9228 100644
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmMultipartKeyInfo.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmMultipartKeyInfo.java
@@ -20,6 +20,7 @@
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.MultipartKeyInfo;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.PartKeyInfo;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.FileHandle;
import java.util.HashMap;
import java.util.Map;
@@ -35,14 +36,17 @@
private final ReplicationType replicationType;
private final ReplicationFactor replicationFactor;
private TreeMap<Integer, PartKeyInfo> partKeyInfoList;
+ private long fileHandleInfo;
/**
* Construct OmMultipartKeyInfo object which holds multipart upload
* information for a key.
*/
+ @SuppressWarnings("parameternumber")
public OmMultipartKeyInfo(String id, long creationTime,
ReplicationType replicationType, ReplicationFactor replicationFactor,
- Map<Integer, PartKeyInfo> list, long objectID, long updateID) {
+ Map<Integer, PartKeyInfo> list, long objectID, long updateID,
+ long fileHandleInfo) {
this.uploadID = id;
this.creationTime = creationTime;
this.replicationType = replicationType;
@@ -50,6 +54,19 @@
this.partKeyInfoList = new TreeMap<>(list);
this.objectID = objectID;
this.updateID = updateID;
+ if (fileHandleInfo != 0) {
+ this.fileHandleInfo = fileHandleInfo;
+ } else {
+ this.fileHandleInfo = objectID;
+ }
+ }
+
+ public long getFileHandleInfo() {
+ return fileHandleInfo;
+ }
+
+ public void setFileHandleInfo(long fileHandleInfo) {
+ this.fileHandleInfo = fileHandleInfo;
}
/**
@@ -95,6 +112,7 @@
private TreeMap<Integer, PartKeyInfo> partKeyInfoList;
private long objectID;
private long updateID;
+ private long fileHandleInfo;
public Builder() {
this.partKeyInfoList = new TreeMap<>();
@@ -139,6 +157,11 @@
return this;
}
+ public Builder setFileHandleInfo(long fh) {
+ this.fileHandleInfo = fh;
+ return this;
+ }
+
public Builder setUpdateID(long id) {
this.updateID = id;
return this;
@@ -146,7 +169,8 @@
public OmMultipartKeyInfo build() {
return new OmMultipartKeyInfo(uploadID, creationTime, replicationType,
- replicationFactor, partKeyInfoList, objectID, updateID);
+ replicationFactor, partKeyInfoList, objectID, updateID,
+ fileHandleInfo);
}
}
@@ -160,10 +184,17 @@
Map<Integer, PartKeyInfo> list = new HashMap<>();
multipartKeyInfo.getPartKeyInfoListList().forEach(partKeyInfo ->
list.put(partKeyInfo.getPartNumber(), partKeyInfo));
+ long fhid = 0;
+ if (multipartKeyInfo.hasFileHandleInfo()) {
+ FileHandle fh = multipartKeyInfo.getFileHandleInfo();
+ if (fh.hasFhObjectID()) {
+ fhid = fh.getFhObjectID();
+ }
+ }
return new OmMultipartKeyInfo(multipartKeyInfo.getUploadID(),
multipartKeyInfo.getCreationTime(), multipartKeyInfo.getType(),
multipartKeyInfo.getFactor(), list, multipartKeyInfo.getObjectID(),
- multipartKeyInfo.getUpdateID());
+ multipartKeyInfo.getUpdateID(), fhid);
}
/**
@@ -171,13 +202,16 @@
* @return MultipartKeyInfo
*/
public MultipartKeyInfo getProto() {
+ FileHandle.Builder fb = FileHandle.newBuilder()
+ .setFhObjectID(fileHandleInfo);
MultipartKeyInfo.Builder builder = MultipartKeyInfo.newBuilder()
.setUploadID(uploadID)
.setCreationTime(creationTime)
.setType(replicationType)
.setFactor(replicationFactor)
.setObjectID(objectID)
- .setUpdateID(updateID);
+ .setUpdateID(updateID)
+ .setFileHandleInfo(fb.build());
partKeyInfoList.forEach((key, value) -> builder.addPartKeyInfoList(value));
return builder.build();
}
@@ -205,7 +239,8 @@
// For partKeyInfoList we can do shallow copy here, as the PartKeyInfo is
// immutable here.
return new OmMultipartKeyInfo(uploadID, creationTime, replicationType,
- replicationFactor, new TreeMap<>(partKeyInfoList), objectID, updateID);
+ replicationFactor, new TreeMap<>(partKeyInfoList), objectID, updateID,
+ fileHandleInfo);
}
}
diff --git a/hadoop-ozone/common/src/main/proto/OzoneManagerProtocol.proto b/hadoop-ozone/common/src/main/proto/OzoneManagerProtocol.proto
index 569be74..77437e7 100644
--- a/hadoop-ozone/common/src/main/proto/OzoneManagerProtocol.proto
+++ b/hadoop-ozone/common/src/main/proto/OzoneManagerProtocol.proto
@@ -719,6 +719,11 @@
optional FileEncryptionInfoProto fileEncryptionInfo = 3;
}
+message FileHandle {
+ optional uint64 fhObjectID = 1;
+ optional string fhObjectStr = 2;
+}
+
message KeyInfo {
required string volumeName = 1;
required string bucketName = 2;
@@ -735,6 +740,7 @@
repeated OzoneAclInfo acls = 13;
optional uint64 objectID = 14;
optional uint64 updateID = 15;
+ optional FileHandle fileHandleInfo = 16;
}
message RepeatedKeyInfo {
@@ -992,6 +998,7 @@
repeated PartKeyInfo partKeyInfoList = 5;
optional uint64 objectID = 6;
optional uint64 updateID = 7;
+ optional FileHandle fileHandleInfo = 8;
}
message PartKeyInfo {
diff --git a/hadoop-ozone/common/src/main/proto/proto.lock b/hadoop-ozone/common/src/main/proto/proto.lock
index 0de831c..fcdc830 100644
--- a/hadoop-ozone/common/src/main/proto/proto.lock
+++ b/hadoop-ozone/common/src/main/proto/proto.lock
@@ -2060,6 +2060,21 @@
]
},
{
+ "name": "FileHandle",
+ "fields": [
+ {
+ "id": 1,
+ "name": "fhObjectID",
+ "type": "uint64"
+ },
+ {
+ "id": 2,
+ "name": "fhObjectStr",
+ "type": "string"
+ }
+ ]
+ },
+ {
"name": "KeyInfo",
"fields": [
{
@@ -2139,6 +2154,11 @@
"id": 15,
"name": "updateID",
"type": "uint64"
+ },
+ {
+ "id": 16,
+ "name": "fileHandleInfo",
+ "type": "FileHandle"
}
]
},
@@ -2828,6 +2848,11 @@
"id": 7,
"name": "updateID",
"type": "uint64"
+ },
+ {
+ "id": 8,
+ "name": "fileHandleInfo",
+ "type": "FileHandle"
}
]
},
diff --git a/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmKeyInfo.java b/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmKeyInfo.java
index eb53916..73c3f7e 100644
--- a/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmKeyInfo.java
+++ b/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmKeyInfo.java
@@ -56,6 +56,8 @@
.setReplicationType(ReplicationType.RATIS)
.addMetadata("key1", "value1")
.addMetadata("key2", "value2")
+ .addMetadata("key2", "value2")
+ .setFileHandleInfo(1234L)
.build();
OmKeyInfo keyAfterSerialization =
@@ -79,10 +81,13 @@
.addMetadata("key2", "value2")
.setOmKeyLocationInfos(
Collections.singletonList(createOmKeyLocationInfoGroup()))
+ .setFileHandleInfo(1234L)
.build();
OmKeyInfo cloneKey = key.copyObject();
+ Assert.assertEquals(key.getFileHandleInfo(), cloneKey.getFileHandleInfo());
+
// Because for OmKeyLocationInfoGroup we have not implemented equals()
// method, so it checks only references.
Assert.assertNotEquals(key, cloneKey);
diff --git a/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmMultipartKeyInfo.java b/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmMultipartKeyInfo.java
index 599a563..10c810f 100644
--- a/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmMultipartKeyInfo.java
+++ b/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmMultipartKeyInfo.java
@@ -39,11 +39,14 @@
.setCreationTime(Time.now())
.setReplicationType(HddsProtos.ReplicationType.RATIS)
.setReplicationFactor(HddsProtos.ReplicationFactor.THREE)
+ .setFileHandleInfo(1234L)
.build();
OmMultipartKeyInfo cloneMultipartKeyInfo = omMultipartKeyInfo.copyObject();
Assert.assertEquals(cloneMultipartKeyInfo, omMultipartKeyInfo);
+ Assert.assertEquals(cloneMultipartKeyInfo.getFileHandleInfo(),
+ omMultipartKeyInfo.getFileHandleInfo());
// Just setting dummy values for this test.
omMultipartKeyInfo.addPartKeyInfo(1,
