TEZ-2907. NPE in IFile.Reader.getLength during final merge operation (rbalamohan)
(cherry picked from commit f3cea630e8329018a1e600859353576cb83cf77d)
# Conflicts:
# CHANGES.txt
diff --git a/CHANGES.txt b/CHANGES.txt
index 0356b75..a91aa7f 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -6,6 +6,7 @@
INCOMPATIBLE CHANGES
ALL CHANGES:
+ TEZ-2907. NPE in IFile.Reader.getLength during final merge operation
TEZ-2850. Tez MergeManager OOM for small Map Outputs
TEZ-2781. Fallback to send only TaskAttemptFailedEvent if taskFailed heartbeat fails
TEZ-2855. Fix a potential NPE while routing VertexManager events.
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/MergeManager.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/MergeManager.java
index efecb8a..f498787 100644
--- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/MergeManager.java
+++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/MergeManager.java
@@ -812,12 +812,14 @@
class RawKVIteratorReader extends IFile.Reader {
private final TezRawKeyValueIterator kvIter;
+ private final long size;
public RawKVIteratorReader(TezRawKeyValueIterator kvIter, long size)
throws IOException {
super(null, size, null, spilledRecordsCounter, null, ifileReadAhead,
ifileReadAheadLength, ifileBufferSize);
this.kvIter = kvIter;
+ this.size = size;
}
@Override
public KeyState readRawKey(DataInputBuffer key) throws IOException {
@@ -845,6 +847,10 @@
public void close() throws IOException {
kvIter.close();
}
+
+ @Override public long getLength() {
+ return size;
+ }
}
private TezRawKeyValueIterator finalMerge(Configuration job, FileSystem fs,
