TEZ-4296 - Use listStatusIterator instead of listStatus in DatePartitionedLogger (#124)
Co-authored-by: Harish JP <hjp@cloudera.com>
diff --git a/tez-plugins/tez-protobuf-history-plugin/src/main/java/org/apache/tez/dag/history/logging/proto/DatePartitionedLogger.java b/tez-plugins/tez-protobuf-history-plugin/src/main/java/org/apache/tez/dag/history/logging/proto/DatePartitionedLogger.java
index 4ac64c6..a569567 100644
--- a/tez-plugins/tez-protobuf-history-plugin/src/main/java/org/apache/tez/dag/history/logging/proto/DatePartitionedLogger.java
+++ b/tez-plugins/tez-protobuf-history-plugin/src/main/java/org/apache/tez/dag/history/logging/proto/DatePartitionedLogger.java
@@ -32,6 +32,7 @@
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.fs.RemoteIterator;
import org.apache.hadoop.yarn.util.Clock;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -138,7 +139,9 @@
}
// Have to scan the directory to find min date greater than currentDir.
String dirName = null;
- for (FileStatus status : fileSystem.listStatus(basePath)) {
+ RemoteIterator<FileStatus> iter = fileSystem.listStatusIterator(basePath);
+ while (iter.hasNext()) {
+ FileStatus status = iter.next();
String name = status.getPath().getName();
// String comparison is good enough, since its of form date=yyyy-MM-dd
if (name.compareTo(currentDir) > 0 && (dirName == null || name.compareTo(dirName) < 0)) {
@@ -160,7 +163,9 @@
if (!fileSystem.exists(dirPath)) {
return newFiles;
}
- for (FileStatus status : fileSystem.listStatus(dirPath)) {
+ RemoteIterator<FileStatus> iter = fileSystem.listStatusIterator(dirPath);
+ while (iter.hasNext()) {
+ FileStatus status = iter.next();
String fileName = status.getPath().getName();
Long offset = currentOffsets.get(fileName);
// If the offset was never added or offset < fileSize.
diff --git a/tez-plugins/tez-protobuf-history-plugin/src/test/java/org/apache/tez/dag/history/logging/proto/TestProtoHistoryLoggingService.java b/tez-plugins/tez-protobuf-history-plugin/src/test/java/org/apache/tez/dag/history/logging/proto/TestProtoHistoryLoggingService.java
index 143faed..fd3154d 100644
--- a/tez-plugins/tez-protobuf-history-plugin/src/test/java/org/apache/tez/dag/history/logging/proto/TestProtoHistoryLoggingService.java
+++ b/tez-plugins/tez-protobuf-history-plugin/src/test/java/org/apache/tez/dag/history/logging/proto/TestProtoHistoryLoggingService.java
@@ -277,6 +277,7 @@
service.setAppContext(appContext);
Configuration conf = new Configuration(false);
String basePath = tempFolder.newFolder().getAbsolutePath();
+ conf.set("fs.file.impl", "org.apache.hadoop.fs.RawLocalFileSystem");
conf.set(TezConfiguration.TEZ_HISTORY_LOGGING_PROTO_BASE_DIR, basePath);
conf.setBoolean(TezConfiguration.TEZ_HISTORY_LOGGING_PROTO_SPLIT_DAG_START, splitEvents);
service.init(conf);
