[BUG] Fix bookie port conflict when using LocalBookKeeper
### Motivation
When using `bin/bookkeeper localbookie 10`, always output following exception, and run failed.
```
java.net.BindException: Address already in use
at sun.nio.ch.Net.bind0(Native Method)
at sun.nio.ch.Net.bind(Net.java:433)
at sun.nio.ch.Net.bind(Net.java:425)
at sun.nio.ch.ServerSocketChannelImpl.bind(ServerSocketChannelImpl.java:223)
at io.netty.channel.socket.nio.NioServerSocketChannel.doBind(NioServerSocketChannel.java:130)
at io.netty.channel.AbstractChannel$AbstractUnsafe.bind(AbstractChannel.java:562)
at io.netty.channel.DefaultChannelPipeline$HeadContext.bind(DefaultChannelPipeline.java:1358)
at io.netty.channel.AbstractChannelHandlerContext.invokeBind(AbstractChannelHandlerContext.java:501)
at io.netty.channel.AbstractChannelHandlerContext.bind(AbstractChannelHandlerContext.java:486)
at io.netty.channel.DefaultChannelPipeline.bind(DefaultChannelPipeline.java:1019)
at io.netty.channel.AbstractChannel.bind(AbstractChannel.java:258)
at io.netty.bootstrap.AbstractBootstrap$2.run(AbstractBootstrap.java:366)
at io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:163)
at io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:404)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:474)
at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909)
at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
at java.lang.Thread.run(Thread.java:748)
```
### Changes
Use `PortManager` to obtain available ports based on initialPort.
Reviewers: Enrico Olivelli <eolivelli@gmail.com>, Sijie Guo <None>
This closes #2338 from lamber-ken/fix-localbookeeper-error
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/util/LocalBookKeeper.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/util/LocalBookKeeper.java
index 36cf72b..1c87284 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/util/LocalBookKeeper.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/util/LocalBookKeeper.java
@@ -87,7 +87,8 @@
this.initialPort = initialPort;
this.localBookiesConfigDir = new File(localBookiesConfigDirName);
this.baseConf = baseConf;
- LOG.info("Running {} bookie(s) on zkServer {}.", this.numberOfBookies);
+ LOG.info("Running {} bookie(s) on zk ensemble = '{}:{}'.", this.numberOfBookies,
+ zooKeeperDefaultHost, zooKeeperDefaultPort);
}
private static String zooKeeperDefaultHost = "127.0.0.1";
@@ -238,10 +239,11 @@
// If the caller specified ephemeral ports then use ephemeral ports for all
// the bookies else use numBookie ports starting at initialPort
+ PortManager.initPort(initialPort);
if (0 == initialPort) {
bsConfs[i].setBookiePort(0);
} else {
- bsConfs[i].setBookiePort(initialPort + i);
+ bsConfs[i].setBookiePort(PortManager.nextFreePort());
}
if (null == baseConf.getMetadataServiceUriUnchecked()) {
@@ -451,10 +453,10 @@
String confFile = args[1];
try {
conf.loadConf(new File(confFile).toURI().toURL());
- LOG.info("Using configuration file " + confFile);
+ LOG.info("Using configuration file {}", confFile);
} catch (Exception e) {
// load conf failed
- LOG.warn("Error loading configuration file " + confFile, e);
+ LOG.warn("Error loading configuration file {}", confFile, e);
}
}
@@ -472,7 +474,6 @@
bookieDefaultInitialPort, false, "test", zkDataDir, localBookiesConfigDirName);
} catch (Exception e) {
LOG.error("Exiting LocalBookKeeper because of exception in main method", e);
- e.printStackTrace();
/*
* This is needed because, some non-daemon thread (probably in ZK or
* some other dependent service) is preventing the JVM from exiting, though
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/util/PortManager.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/util/PortManager.java
new file mode 100644
index 0000000..864aaf2
--- /dev/null
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/util/PortManager.java
@@ -0,0 +1,74 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+package org.apache.bookkeeper.util;
+
+import java.io.IOException;
+import java.net.ServerSocket;
+import java.util.concurrent.TimeUnit;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Port manager allows a base port to be specified on the commandline.
+ * Tests will then use ports, counting up from this base port.
+ * This allows multiple instances of the bookkeeper tests to run at once.
+ */
+public class PortManager {
+
+ private static final Logger LOG = LoggerFactory.getLogger(PortManager.class);
+
+ private static int nextPort = 15000;
+
+ /**
+ * Init the base port.
+ *
+ * @param initPort initial port
+ */
+ public static void initPort(int initPort) {
+ nextPort = initPort;
+ }
+
+ /**
+ * Return the available port.
+ *
+ * @return available port.
+ */
+ public static synchronized int nextFreePort() {
+ int exceptionCount = 0;
+ while (true) {
+ int port = nextPort++;
+ try (ServerSocket ignored = new ServerSocket(port)) {
+ // Give it some time to truly close the connection
+ TimeUnit.MILLISECONDS.sleep(100);
+ return port;
+ } catch (IOException ioe) {
+ exceptionCount++;
+ if (exceptionCount > 100) {
+ throw new RuntimeException("Unable to allocate socket port", ioe);
+ }
+ } catch (InterruptedException ie) {
+ LOG.error("Failed to allocate socket port", ie);
+ Thread.currentThread().interrupt();
+ }
+ }
+ }
+
+}
diff --git a/bookkeeper-server/src/main/resources/log4j.properties b/bookkeeper-server/src/main/resources/log4j.properties
new file mode 100644
index 0000000..7703c57
--- /dev/null
+++ b/bookkeeper-server/src/main/resources/log4j.properties
@@ -0,0 +1,38 @@
+#
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#
+
+#
+# Bookkeeper Logging Configuration
+#
+
+# Format is "<default threshold> (, <appender>)+
+
+# DEFAULT: console appender only, level INFO
+bookkeeper.root.logger=INFO,CONSOLE
+log4j.rootLogger=${bookkeeper.root.logger}
+
+#
+# Log INFO level and above messages to the console
+#
+log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender
+log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout
+log4j.appender.CONSOLE.layout.ConversionPattern=%d{ISO8601} - %-5p - [%t:%C{1}@%L] - %m%n
+
diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/AdvertisedAddressTest.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/AdvertisedAddressTest.java
index 0f893ac..72e59b1 100644
--- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/AdvertisedAddressTest.java
+++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/AdvertisedAddressTest.java
@@ -32,8 +32,8 @@
import org.apache.bookkeeper.conf.TestBKConfiguration;
import org.apache.bookkeeper.net.BookieSocketAddress;
import org.apache.bookkeeper.test.BookKeeperClusterTestCase;
-import org.apache.bookkeeper.test.PortManager;
import org.apache.bookkeeper.util.IOUtils;
+import org.apache.bookkeeper.util.PortManager;
import org.junit.Test;
/**
diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/BookieInitializationTest.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/BookieInitializationTest.java
index 25df88f..f57d268 100644
--- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/BookieInitializationTest.java
+++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/BookieInitializationTest.java
@@ -110,10 +110,10 @@
import org.apache.bookkeeper.stats.StatsLogger;
import org.apache.bookkeeper.stats.prometheus.PrometheusMetricsProvider;
import org.apache.bookkeeper.test.BookKeeperClusterTestCase;
-import org.apache.bookkeeper.test.PortManager;
import org.apache.bookkeeper.tls.SecurityException;
import org.apache.bookkeeper.util.DiskChecker;
import org.apache.bookkeeper.util.LoggerOutput;
+import org.apache.bookkeeper.util.PortManager;
import org.apache.bookkeeper.versioning.Version;
import org.apache.bookkeeper.versioning.Versioned;
import org.apache.bookkeeper.zookeeper.ZooKeeperClient;
diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/CookieTest.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/CookieTest.java
index e91500f..e244852 100644
--- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/CookieTest.java
+++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/CookieTest.java
@@ -45,9 +45,9 @@
import org.apache.bookkeeper.meta.MetadataDrivers;
import org.apache.bookkeeper.stats.NullStatsLogger;
import org.apache.bookkeeper.test.BookKeeperClusterTestCase;
-import org.apache.bookkeeper.test.PortManager;
import org.apache.bookkeeper.util.BookKeeperConstants;
import org.apache.bookkeeper.util.IOUtils;
+import org.apache.bookkeeper.util.PortManager;
import org.apache.bookkeeper.versioning.LongVersion;
import org.apache.bookkeeper.versioning.Version;
import org.apache.bookkeeper.versioning.Versioned;
diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/LedgerStorageCheckpointTest.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/LedgerStorageCheckpointTest.java
index 1ea661b..9d69f3e 100644
--- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/LedgerStorageCheckpointTest.java
+++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/LedgerStorageCheckpointTest.java
@@ -53,9 +53,9 @@
import org.apache.bookkeeper.conf.ServerConfiguration;
import org.apache.bookkeeper.conf.TestBKConfiguration;
import org.apache.bookkeeper.proto.BookieServer;
-import org.apache.bookkeeper.test.PortManager;
import org.apache.bookkeeper.test.ZooKeeperUtil;
import org.apache.bookkeeper.util.IOUtils;
+import org.apache.bookkeeper.util.PortManager;
import org.apache.commons.io.FileUtils;
import org.junit.After;
import org.junit.Assert;
diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/UpgradeTest.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/UpgradeTest.java
index 0ddd9a6..b49950b 100644
--- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/UpgradeTest.java
+++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/UpgradeTest.java
@@ -42,8 +42,8 @@
import org.apache.bookkeeper.conf.ServerConfiguration;
import org.apache.bookkeeper.conf.TestBKConfiguration;
import org.apache.bookkeeper.test.BookKeeperClusterTestCase;
-import org.apache.bookkeeper.test.PortManager;
import org.apache.bookkeeper.util.IOUtils;
+import org.apache.bookkeeper.util.PortManager;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/test/BookKeeperClusterTestCase.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/test/BookKeeperClusterTestCase.java
index 3d9e88f..afc8302 100644
--- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/test/BookKeeperClusterTestCase.java
+++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/test/BookKeeperClusterTestCase.java
@@ -61,6 +61,7 @@
import org.apache.bookkeeper.replication.ReplicationException.CompatibilityException;
import org.apache.bookkeeper.replication.ReplicationException.UnavailableException;
import org.apache.bookkeeper.util.IOUtils;
+import org.apache.bookkeeper.util.PortManager;
import org.apache.commons.io.FileUtils;
import org.apache.zookeeper.KeeperException;
import org.apache.zookeeper.ZooKeeper;
diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/test/BookieZKExpireTest.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/test/BookieZKExpireTest.java
index 37160b5..3751401 100644
--- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/test/BookieZKExpireTest.java
+++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/test/BookieZKExpireTest.java
@@ -30,6 +30,7 @@
import org.apache.bookkeeper.conf.ServerConfiguration;
import org.apache.bookkeeper.proto.BookieServer;
+import org.apache.bookkeeper.util.PortManager;
import org.junit.Test;
/**
diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/test/PortManager.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/test/PortManager.java
deleted file mode 100644
index 0a4b8ab..0000000
--- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/test/PortManager.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- */
-package org.apache.bookkeeper.test;
-
-import java.net.ServerSocket;
-/**
- * Port manager allows a base port to be specified on the commandline.
- * Tests will then use ports, counting up from this base port.
- * This allows multiple instances of the bookkeeper tests to run at once.
- */
-public class PortManager {
- private static int nextPort = getBasePort();
-
- public static synchronized int nextFreePort() {
- int exceptionCount = 0;
- while (true) {
- int port = nextPort++;
- try (ServerSocket ss = new ServerSocket(port)) {
- ss.close();
- //Give it some time to truly close the connection
- Thread.sleep(100);
- return port;
- } catch (Exception e) {
- exceptionCount++;
- if (exceptionCount > 5) {
- throw new RuntimeException(e);
- }
- }
- }
- }
-
- private static int getBasePort() {
- return Integer.valueOf(System.getProperty("test.basePort", "15000"));
- }
-}
diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/test/ReadOnlyBookieTest.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/test/ReadOnlyBookieTest.java
index 329779c..a154121 100644
--- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/test/ReadOnlyBookieTest.java
+++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/test/ReadOnlyBookieTest.java
@@ -35,6 +35,7 @@
import org.apache.bookkeeper.client.LedgerEntry;
import org.apache.bookkeeper.client.LedgerHandle;
import org.apache.bookkeeper.conf.ServerConfiguration;
+import org.apache.bookkeeper.util.PortManager;
import org.junit.Test;
/**
diff --git a/metadata-drivers/etcd/src/test/java/org/apache/bookkeeper/metadata/etcd/testing/EtcdBKClusterTestBase.java b/metadata-drivers/etcd/src/test/java/org/apache/bookkeeper/metadata/etcd/testing/EtcdBKClusterTestBase.java
index 7ccb0f0..0c3cebf 100644
--- a/metadata-drivers/etcd/src/test/java/org/apache/bookkeeper/metadata/etcd/testing/EtcdBKClusterTestBase.java
+++ b/metadata-drivers/etcd/src/test/java/org/apache/bookkeeper/metadata/etcd/testing/EtcdBKClusterTestBase.java
@@ -32,9 +32,9 @@
import org.apache.bookkeeper.metadata.etcd.EtcdMetadataBookieDriver;
import org.apache.bookkeeper.metadata.etcd.EtcdMetadataClientDriver;
import org.apache.bookkeeper.proto.BookieServer;
-import org.apache.bookkeeper.test.PortManager;
import org.apache.bookkeeper.test.TestStatsProvider;
import org.apache.bookkeeper.util.IOUtils;
+import org.apache.bookkeeper.util.PortManager;
import org.apache.commons.io.FileUtils;
import org.junit.After;
import org.junit.AfterClass;
