TEZ-4577: SortSpan could be created real small, resulting in eventual job failure (#367) (Chenyu Zheng reviewed by Laszlo Bodor)
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/PipelinedSorter.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/PipelinedSorter.java
index 1815006..d0eeadc 100644
--- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/PipelinedSorter.java
+++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/PipelinedSorter.java
@@ -357,7 +357,7 @@
}
Preconditions.checkArgument(buffers.get(bufferIndex) != null, "block should not be empty");
//TODO: fix per item being passed.
- span = new SortSpan((ByteBuffer)buffers.get(bufferIndex).clear(), items,
+ span = new SortSpan((ByteBuffer)buffers.get(bufferIndex).clear(), (1024*1024),
perItem, ConfigUtils.getIntermediateOutputKeyComparator(this.conf));
} else {
// queue up the sort
@@ -940,7 +940,7 @@
public SortSpan(ByteBuffer source, int maxItems, int perItem, RawComparator comparator) {
capacity = source.remaining();
int metasize = METASIZE*maxItems;
- int dataSize = maxItems * perItem;
+ long dataSize = (long) maxItems * (long) perItem;
if(capacity < (metasize+dataSize)) {
// try to allocate less meta space, because we have sample data
metasize = METASIZE*(capacity/(perItem+METASIZE));