commit fe7c9e71eb0aa365306c2d91da82234a96a5c1c6
author Prashant Wason <pwason@uber.com> Mon Sep 07 19:30:45 2020 -0700
committer GitHub <noreply@github.com> Mon Sep 07 19:30:45 2020 -0700
tree de2e4814472076d8b1c9c0e976da4aaf5161617b
parent 83e39e2b179fb1db13f1c4865a1849855fc32b65

[MINOR] Fix BindException when running tests of shared machines. (#2070)

When unit tests are run on shared machines (e.g. jenkins cluster), the unit tests sometimes fail due to BindException in starting HDFS Cluster. This is because the port chosen may have been bound by another process using the same machine. The fix is to retry the port selection a few times.

hudi-common/src/test/java/org/apache/hudi/common/testutils/minicluster/HdfsTestService.java

diff --git a/hudi-common/src/test/java/org/apache/hudi/common/testutils/minicluster/HdfsTestService.java b/hudi-common/src/test/java/org/apache/hudi/common/testutils/minicluster/HdfsTestService.java
index 44819d3..44af4ec 100644
--- a/hudi-common/src/test/java/org/apache/hudi/common/testutils/minicluster/HdfsTestService.java
+++ b/hudi-common/src/test/java/org/apache/hudi/common/testutils/minicluster/HdfsTestService.java
@@ -31,6 +31,7 @@
 
 import java.io.File;
 import java.io.IOException;
+import java.net.BindException;
 import java.nio.file.Files;
 import java.util.Objects;
 
@@ -72,20 +73,32 @@
       FileIOUtils.deleteDirectory(file);
     }
 
-    int namenodeRpcPort = NetworkTestUtils.nextFreePort();
-    int datanodePort = NetworkTestUtils.nextFreePort();
-    int datanodeIpcPort = NetworkTestUtils.nextFreePort();
-    int datanodeHttpPort = NetworkTestUtils.nextFreePort();
+    int loop = 0;
+    while (true) {
+      try {
+        int namenodeRpcPort = NetworkTestUtils.nextFreePort();
+        int datanodePort = NetworkTestUtils.nextFreePort();
+        int datanodeIpcPort = NetworkTestUtils.nextFreePort();
+        int datanodeHttpPort = NetworkTestUtils.nextFreePort();
 
-    // Configure and start the HDFS cluster
-    // boolean format = shouldFormatDFSCluster(localDFSLocation, clean);
-    String bindIP = "127.0.0.1";
-    configureDFSCluster(hadoopConf, localDFSLocation, bindIP, namenodeRpcPort,
-        datanodePort, datanodeIpcPort, datanodeHttpPort);
-    miniDfsCluster = new MiniDFSCluster.Builder(hadoopConf).numDataNodes(1).format(format).checkDataNodeAddrConfig(true)
-        .checkDataNodeHostConfig(true).build();
-    LOG.info("HDFS Minicluster service started.");
-    return miniDfsCluster;
+        // Configure and start the HDFS cluster
+        // boolean format = shouldFormatDFSCluster(localDFSLocation, clean);
+        String bindIP = "127.0.0.1";
+        configureDFSCluster(hadoopConf, localDFSLocation, bindIP, namenodeRpcPort,
+            datanodePort, datanodeIpcPort, datanodeHttpPort);
+        miniDfsCluster = new MiniDFSCluster.Builder(hadoopConf).numDataNodes(1).format(format).checkDataNodeAddrConfig(true)
+            .checkDataNodeHostConfig(true).build();
+        LOG.info("HDFS Minicluster service started.");
+        return miniDfsCluster;
+      } catch (BindException ex) {
+        ++loop;
+        if (loop < 5) {
+          stop();
+        } else {
+          throw ex;
+        }
+      }
+    }
   }
 
   public void stop() {