Returning empty Statues for an empty spark partition caused due to incorrect bin packing
diff --git a/hoodie-client/src/main/java/com/uber/hoodie/table/HoodieCopyOnWriteTable.java b/hoodie-client/src/main/java/com/uber/hoodie/table/HoodieCopyOnWriteTable.java
index fc6dcf9..0a7eeae 100644
--- a/hoodie-client/src/main/java/com/uber/hoodie/table/HoodieCopyOnWriteTable.java
+++ b/hoodie-client/src/main/java/com/uber/hoodie/table/HoodieCopyOnWriteTable.java
@@ -176,6 +176,11 @@
public Iterator<List<WriteStatus>> handleUpdate(String commitTime, String fileId,
Iterator<HoodieRecord<T>> recordItr) throws IOException {
+ // This is needed since sometimes some buckets are never picked in getPartition() and end up with 0 records
+ if (!recordItr.hasNext()) {
+ logger.info("Empty partition with fileId => " + fileId);
+ return Collections.singletonList((List<WriteStatus>) Collections.EMPTY_LIST).iterator();
+ }
// these are updates
HoodieMergeHandle upsertHandle = getUpdateHandle(commitTime, fileId, recordItr);
return handleUpdateInternal(upsertHandle, commitTime, fileId);
@@ -235,6 +240,11 @@
public Iterator<List<WriteStatus>> handleInsert(String commitTime,
Iterator<HoodieRecord<T>> recordItr) throws Exception {
+ // This is needed since sometimes some buckets are never picked in getPartition() and end up with 0 records
+ if (!recordItr.hasNext()) {
+ logger.info("Empty partition");
+ return Collections.singletonList((List<WriteStatus>) Collections.EMPTY_LIST).iterator();
+ }
return new CopyOnWriteLazyInsertIterable<>(recordItr, config, commitTime, this);
}
