SLIDER-82 trying to set up YarnClient, having config propagation issues in minicluster (RM client addr is 0.0.0.0)
diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java b/slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java
index 803ccd6..867603b 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java
@@ -45,6 +45,7 @@
import java.io.IOException;
import java.net.BindException;
+import java.net.InetSocketAddress;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashSet;
@@ -69,10 +70,11 @@
@Override
protected void serviceInit(Configuration conf) throws Exception {
- String addr = conf.get(YarnConfiguration.RM_ADDRESS);
- if (addr.startsWith("0.0.0.0")) {
+ InetSocketAddress clientRpcAddress = SliderUtils.getRmAddress(conf);
+ if (!SliderUtils.isAddressDefined(clientRpcAddress)) {
// address isn't known; fail fast
- throw new BindException("Invalid " + YarnConfiguration.RM_ADDRESS + " value:" + addr
+ throw new BindException("Invalid " + YarnConfiguration.RM_ADDRESS
+ + " value:" + conf.get(YarnConfiguration.RM_ADDRESS)
+ " - see https://wiki.apache.org/hadoop/UnsetHostnameOrPort");
}
super.serviceInit(conf);
diff --git a/slider-core/src/main/java/org/apache/slider/core/launch/ContainerLauncher.java b/slider-core/src/main/java/org/apache/slider/core/launch/ContainerLauncher.java
index f8ea4ee..69b937d 100644
--- a/slider-core/src/main/java/org/apache/slider/core/launch/ContainerLauncher.java
+++ b/slider-core/src/main/java/org/apache/slider/core/launch/ContainerLauncher.java
@@ -55,17 +55,13 @@
public UserGroupInformation setupUGI() {
UserGroupInformation user =
UserGroupInformation.createRemoteUser(container.getId().toString());
- String cmIpPortStr =
- container.getNodeId().getHost() + ":" + container.getNodeId().getPort();
- final InetSocketAddress cmAddress =
- NetUtils.createSocketAddr(cmIpPortStr);
+ String cmIpPortStr = container.getNodeId().getHost() + ":" + container.getNodeId().getPort();
+ final InetSocketAddress cmAddress = NetUtils.createSocketAddr(cmIpPortStr);
- org.apache.hadoop.yarn.api.records.Token containerToken =
- container.getContainerToken();
+ org.apache.hadoop.yarn.api.records.Token containerToken = container.getContainerToken();
if (containerToken != null) {
Token<ContainerTokenIdentifier> token =
- ConverterUtils.convertFromYarn(containerToken,
- cmAddress);
+ ConverterUtils.convertFromYarn(containerToken, cmAddress);
user.addToken(token);
}
return user;
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index e6a5bd5..1a127cf 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -502,15 +502,6 @@
addService(executorService);
addService(actionQueues);
- // set up the YARN client. This may require patching in the RM client-API address if it
- // is (somehow) unset server-side.
- String clientRMaddr = conf.get(YarnConfiguration.RM_ADDRESS);
- if (clientRMaddr.startsWith("0.0.0.0")) {
- // address isn't known; fail fast
- throw new BindException("Invalid " + YarnConfiguration.RM_ADDRESS + " value:" + addr
- + " - see https://wiki.apache.org/hadoop/UnsetHostnameOrPort");
- }
- addService(yarnClient = new SliderYarnClientImpl());
//init all child services
super.serviceInit(conf);
@@ -542,8 +533,7 @@
* @param args argument list
*/
@Override // RunService
- public Configuration bindArgs(Configuration config, String... args) throws
- Exception {
+ public Configuration bindArgs(Configuration config, String... args) throws Exception {
// let the superclass process it
Configuration superConf = super.bindArgs(config, args);
// add the slider XML config
@@ -667,10 +657,21 @@
sliderAMProvider = new SliderAMProviderService();
initAndAddService(sliderAMProvider);
- InetSocketAddress address = SliderUtils.getRmSchedulerAddress(serviceConf);
- log.info("RM is at {}", address);
+ InetSocketAddress rmSchedulerAddress = SliderUtils.getRmSchedulerAddress(serviceConf);
+ log.info("RM is at {}", rmSchedulerAddress);
yarnRPC = YarnRPC.create(serviceConf);
+ // set up the YARN client. This may require patching in the RM client-API address if it
+ // is (somehow) unset server-side. String clientRMaddr = serviceConf.get(YarnConfiguration.RM_ADDRESS);
+ InetSocketAddress clientRpcAddress = SliderUtils.getRmAddress(serviceConf);
+ if (!SliderUtils.isAddressDefined(clientRpcAddress)) {
+ // client addr is being unset. We can lift it from the other RM APIs
+ serviceConf.set(YarnConfiguration.RM_ADDRESS,
+ String.format("%s:%d", rmSchedulerAddress.getHostString(), clientRpcAddress.getPort() ));
+ }
+ initAndAddService(yarnClient = new SliderYarnClientImpl());
+ yarnClient.start();
+
/*
* Extract the container ID. This is then
* turned into an (incompete) container