SLIDER-907 AgentWebPagesIT#testAgentWeb fails on wire encrypted clusters: publish proxied values from outset. This patch also pulls in some changes from SLIDER-82 to set up a YarnClient instance in the AM
diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index fe0e038..b89af4c 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -2327,6 +2327,15 @@
config,
DFSConfigKeys.DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY);
}
+
+ // copy over any/all YARN RM client values, in case the server-side XML conf file
+ // has the 0.0.0.0 address
+ commandLine.addConfOptions(config,
+ YarnConfiguration.RM_ADDRESS,
+ YarnConfiguration.RM_CLUSTER_ID,
+ YarnConfiguration.RM_HOSTNAME,
+ YarnConfiguration.RM_PRINCIPAL);
+
// write out the path output
commandLine.addOutAndErrFiles(STDOUT_AM, STDERR_AM);
diff --git a/slider-core/src/main/java/org/apache/slider/core/launch/JavaCommandLineBuilder.java b/slider-core/src/main/java/org/apache/slider/core/launch/JavaCommandLineBuilder.java
index 0b3fa10..dcb322e 100644
--- a/slider-core/src/main/java/org/apache/slider/core/launch/JavaCommandLineBuilder.java
+++ b/slider-core/src/main/java/org/apache/slider/core/launch/JavaCommandLineBuilder.java
@@ -20,11 +20,15 @@
import com.google.common.base.Preconditions;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.api.ApplicationConstants;
import org.apache.slider.common.tools.SliderUtils;
+import org.apache.slider.core.exceptions.BadConfigException;
/**
- * Command line builder purely for the Java CLI
+ * Command line builder purely for the Java CLI.
+ * Some of the <code>define</code> methods are designed to work with Hadoop tool and
+ * Slider launcher applications.
*/
public class JavaCommandLineBuilder extends CommandLineBuilder {
@@ -80,4 +84,68 @@
sysprop("java.awt.headless", "true");
return this;
}
+
+ public boolean addConfOption(Configuration conf, String key) {
+ return defineIfSet(key, conf.get(key));
+ }
+
+ /**
+ * Add a varargs list of configuration parameters —if they are present
+ * @param conf configuration source
+ * @param keys keys
+ */
+ public void addConfOptions(Configuration conf, String...keys) {
+ for (String key : keys) {
+ addConfOption(conf, key);
+ }
+ }
+
+ public String addConfOptionToCLI(Configuration conf,
+ String key,
+ String defVal) {
+ String val = conf.get(key, defVal);
+ define(key, val);
+ return val;
+ }
+
+ /**
+ * Add a <code>-D key=val</code> command to the CLI. This is very Hadoop API
+ * @param key key
+ * @param val value
+ */
+ public void define(String key, String val) {
+ Preconditions.checkArgument(key != null, "null key");
+ Preconditions.checkArgument(val != null, "null value");
+ add("-D", key + "=" + val);
+ }
+
+ /**
+ * Add a <code>-D key=val</code> command to the CLI if <code>val</code>
+ * is not null
+ * @param key key
+ * @param val value
+ */
+ public boolean defineIfSet(String key, String val) {
+ Preconditions.checkArgument(key != null, "null key");
+ if (val != null) {
+ define(key, val);
+ return true;
+ } else {
+ return false;
+ }
+ }
+
+ /**
+ * Add a mandatory config option
+ * @param conf configuration
+ * @param key key
+ * @throws BadConfigException if the key is missing
+ */
+ public void addMandatoryConfOption(Configuration conf,
+ String key) throws BadConfigException {
+ if (!addConfOption(conf, key)) {
+ throw new BadConfigException("Missing configuration option: " + key);
+ }
+ }
+
}
diff --git a/slider-core/src/main/java/org/apache/slider/providers/ProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/ProviderService.java
index 28cef01..1a18db6 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/ProviderService.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/ProviderService.java
@@ -186,10 +186,10 @@
/**
* Prior to going live -register the initial service registry data
- * @param amWebURI
- * @param agentOpsURI
- * @param agentStatusURI
- * @param serviceRecord
+ * @param amWebURI URL to the AM. This may be proxied, so use relative paths
+ * @param agentOpsURI URI for agent operations. This will not be proxied
+ * @param agentStatusURI URI For agent status. Again: no proxy
+ * @param serviceRecord service record to build up
*/
void applyInitialRegistryDefinitions(URL amWebURI,
URL agentOpsURI,
diff --git a/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMProviderService.java
index cee7a97..be33c21 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMProviderService.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMProviderService.java
@@ -142,10 +142,10 @@
try {
- URL managementAPI = new URL(amWebURI, SLIDER_PATH_MANAGEMENT);
- URL registryREST = new URL(amWebURI, SLIDER_PATH_REGISTRY );
+ URL managementAPI = new URL(amWebURI, RELATIVE_PATH_MANAGEMENT);
+ URL registryREST = new URL(amWebURI, RELATIVE_PATH_REGISTRY);
- URL publisherURL = new URL(amWebURI, SLIDER_PATH_PUBLISHER);
+ URL publisherURL = new URL(amWebURI, RELATIVE_PATH_PUBLISHER);
// Set the configurations URL.
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index 45e2ff9..12709bc 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -47,6 +47,7 @@
import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse;
import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
@@ -82,6 +83,7 @@
import org.apache.slider.api.RoleKeys;
import org.apache.slider.api.StatusKeys;
import org.apache.slider.api.proto.SliderClusterAPI;
+import org.apache.slider.client.SliderYarnClientImpl;
import org.apache.slider.common.SliderExitCodes;
import org.apache.slider.common.SliderKeys;
import org.apache.slider.common.params.AbstractActionArgs;
@@ -273,6 +275,10 @@
@SuppressWarnings("FieldAccessedSynchronizedAndUnsynchronized")
private String appMasterTrackingUrl = "";
+ /** Proxied app master URL (as retrieved from AM report at launch time) */
+ @SuppressWarnings("FieldAccessedSynchronizedAndUnsynchronized")
+ private String appMasterProxiedUrl = "";
+
/** Application Attempt Id ( combination of attemptId and fail count )*/
private ApplicationAttemptId appAttemptID;
@@ -412,6 +418,7 @@
*/
private boolean securityEnabled;
private ContentCache contentCache;
+ private SliderYarnClientImpl yarnClient;
/**
* Service Constructor
@@ -672,10 +679,22 @@
sliderAMProvider = new SliderAMProviderService();
initAndAddService(sliderAMProvider);
- InetSocketAddress address = SliderUtils.getRmSchedulerAddress(serviceConf);
- log.info("RM is at {}", address);
+ InetSocketAddress rmSchedulerAddress = SliderUtils.getRmSchedulerAddress(serviceConf);
+ log.info("RM is at {}", rmSchedulerAddress);
yarnRPC = YarnRPC.create(serviceConf);
+ // set up the YARN client. This may require patching in the RM client-API address if it
+ // is (somehow) unset server-side. String clientRMaddr = serviceConf.get(YarnConfiguration.RM_ADDRESS);
+ InetSocketAddress clientRpcAddress = SliderUtils.getRmAddress(serviceConf);
+ if (!SliderUtils.isAddressDefined(clientRpcAddress)) {
+ // client addr is being unset. We can lift it from the other RM APIs
+ log.warn("Yarn RM address was unbound; attempting to fix up");
+ serviceConf.set(YarnConfiguration.RM_ADDRESS,
+ String.format("%s:%d", rmSchedulerAddress.getHostString(), clientRpcAddress.getPort() ));
+ }
+ initAndAddService(yarnClient = new SliderYarnClientImpl());
+ yarnClient.start();
+
/*
* Extract the container ID. This is then
* turned into an (incompete) container
@@ -830,6 +849,17 @@
}
}
+ // look up the application itself -this is needed to get the proxied
+ // URL of the AM, for registering endpoints.
+ // this call must be made after the AM has registered itself, obviously
+ ApplicationAttemptReport report = yarnClient.getApplicationAttemptReport(
+ appAttemptID);
+ appMasterProxiedUrl = report.getTrackingUrl();
+ if (SliderUtils.isUnset(appMasterProxiedUrl)) {
+ log.warn("Proxied URL is not set in application report");
+ appMasterProxiedUrl = appMasterTrackingUrl;
+ }
+
// extract container list
liveContainers = amRegistrationData.getContainersFromPreviousAttempts();
@@ -954,7 +984,8 @@
// log the YARN and web UIs
log.info("RM Webapp address {}",
serviceConf.get(YarnConfiguration.RM_WEBAPP_ADDRESS));
- log.info("slider Webapp address {}", appMasterTrackingUrl);
+ log.info("Slider webapp address {} proxied at {}",
+ appMasterTrackingUrl, appMasterProxiedUrl);
// Start the Slider AM provider
sliderAMProvider.start();
@@ -1194,7 +1225,7 @@
// the registry is running, so register services
- URL amWebURI = new URL(appMasterTrackingUrl);
+ URL amWebURI = new URL(appMasterProxiedUrl);
URL agentOpsURI = new URL(agentOpsUrl);
URL agentStatusURI = new URL(agentStatusUrl);
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/RestPaths.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/RestPaths.java
index 06b7ba2..17fe31f 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/RestPaths.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/RestPaths.java
@@ -35,8 +35,10 @@
* agent content root: {@value}
*/
public static final String WS_AGENT_CONTEXT_ROOT = "/" + AGENT_WS_CONTEXT;
- public static final String SLIDER_CONTEXT_ROOT = WS_CONTEXT_ROOT +"/v1/slider";
- public static final String SLIDER_AGENT_CONTEXT_ROOT = WS_AGENT_CONTEXT_ROOT +"/v1/slider";
+ public static final String V1_SLIDER = "/v1/slider";
+ public static final String SLIDER_CONTEXT_ROOT = WS_CONTEXT_ROOT + V1_SLIDER;
+ public static final String RELATIVE_API = WS_CONTEXT + V1_SLIDER;
+ public static final String SLIDER_AGENT_CONTEXT_ROOT = WS_AGENT_CONTEXT_ROOT + V1_SLIDER;
public static final String MANAGEMENT = "mgmt";
public static final String SLIDER_SUBPATH_MANAGEMENT = "/" + MANAGEMENT;
public static final String SLIDER_SUBPATH_AGENTS = "/agents";
@@ -49,6 +51,9 @@
public static final String SLIDER_PATH_MANAGEMENT = SLIDER_CONTEXT_ROOT
+ SLIDER_SUBPATH_MANAGEMENT;
+ public static final String RELATIVE_PATH_MANAGEMENT = RELATIVE_API
+ + SLIDER_SUBPATH_MANAGEMENT;
+
/**
* Agents: {@value}
*/
@@ -61,6 +66,9 @@
public static final String SLIDER_PATH_PUBLISHER = SLIDER_CONTEXT_ROOT
+ SLIDER_SUBPATH_PUBLISHER;
+ public static final String RELATIVE_PATH_PUBLISHER = RELATIVE_API
+ + SLIDER_SUBPATH_PUBLISHER;
+
/**
* Registry subpath: {@value}
*/
@@ -71,6 +79,8 @@
*/
public static final String SLIDER_PATH_REGISTRY = SLIDER_CONTEXT_ROOT
+ SLIDER_SUBPATH_REGISTRY;
+ public static final String RELATIVE_PATH_REGISTRY = RELATIVE_API
+ + SLIDER_SUBPATH_REGISTRY;
/**
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneYarnRegistryAM.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneYarnRegistryAM.groovy
index 139860a..56fb5f2 100644
--- a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneYarnRegistryAM.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneYarnRegistryAM.groovy
@@ -18,10 +18,13 @@
package org.apache.slider.agent.standalone
+import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import org.apache.hadoop.fs.FileUtil
import org.apache.hadoop.fs.PathNotFoundException
import org.apache.hadoop.registry.client.binding.RegistryPathUtils
+import org.apache.hadoop.registry.client.types.AddressTypes
+import org.apache.hadoop.registry.client.types.ProtocolTypes
import org.apache.hadoop.yarn.api.records.ApplicationReport
import org.apache.hadoop.yarn.api.records.YarnApplicationState
import org.apache.hadoop.yarn.conf.YarnConfiguration
@@ -31,7 +34,6 @@
import org.apache.hadoop.registry.client.types.RegistryPathStatus
import org.apache.hadoop.registry.client.types.ServiceRecord
import org.apache.hadoop.registry.client.types.yarn.YarnRegistryAttributes
-import org.apache.slider.common.SliderExitCodes
import org.apache.slider.common.params.ActionResolveArgs
import org.apache.slider.common.params.Arguments
import org.apache.slider.core.exceptions.NotFoundException
@@ -59,7 +61,7 @@
/**
* work with a YARN registry
*/
-//@CompileStatic
+@CompileStatic
@Slf4j
class TestStandaloneYarnRegistryAM extends AgentMiniClusterTestBase {
@@ -165,15 +167,19 @@
assert serviceRecord[YarnRegistryAttributes.YARN_PERSISTENCE] != ""
def externalEndpoints = serviceRecord.external;
assert externalEndpoints.size() > 0
-/*
-
- def am_ipc_protocol = AM_IPC_PROTOCOL
- serviceRecord.getExternalEndpoint(am_ipc_protocol)
- assert null != am_ipc_protocol;
-*/
assert null != serviceRecord.getExternalEndpoint(MANAGEMENT_REST_API)
assert null != serviceRecord.getExternalEndpoint(PUBLISHER_REST_API)
+ def publisherEPR = serviceRecord.getExternalEndpoint(PUBLISHER_REST_API)
+ assert publisherEPR.protocolType == ProtocolTypes.PROTOCOL_REST
+ assert publisherEPR.addressType == AddressTypes.ADDRESS_URI
+ def addr = publisherEPR.addresses[0][AddressTypes.ADDRESS_URI]
+ def proxiedURL = new URL(addr)
+ def appId = report.applicationId.toString()
+ // something like http://host:49257/proxy/application_1449844996855_0001/ws/v1/slider/publisher
+ String publisherPath = "/proxy/$appId/" + RestPaths.RELATIVE_PATH_PUBLISHER
+ assert proxiedURL.path == publisherPath
+
// internals
assert null != serviceRecord.getInternalEndpoint(AGENT_ONEWAY_REST_API)
assert null != serviceRecord.getInternalEndpoint(AGENT_SECURE_REST_API)
@@ -323,8 +329,8 @@
String confJSON = GET(yarnSitePublisher)
// log.info(confJSON)
- JsonSerDeser< PublishedConfiguration> confSerDeser =
- new JsonSerDeser<PublishedConfiguration>(PublishedConfiguration)
+ JsonSerDeser<PublishedConfiguration> confSerDeser =
+ new JsonSerDeser<>(PublishedConfiguration)
publishedYarnSite = confSerDeser.fromJson(confJSON)
@@ -488,7 +494,7 @@
assert LauncherExitCodes.EXIT_NOT_FOUND == client.actionRegistry(registryArgs)
//look for a different user
- assertFailsWithException(SliderExitCodes.EXIT_NOT_FOUND, "") {
+ assertFailsWithException(LauncherExitCodes.EXIT_NOT_FOUND, "") {
def args = new ActionRegistryArgs(
name: clustername,
user: "unknown",
