Revert "SLIDER-907 AgentWebPagesIT#testAgentWeb fails on wire encrypted clusters: publish proxied values from outset. This patch also pulls in some changes from SLIDER-82 to set up a YarnClient instance in the AM"
This reverts commit 4ad20fa79fa7228d6edf6e7a18b5a0592d4ee276.
diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index b89af4c..fe0e038 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -2327,15 +2327,6 @@
config,
DFSConfigKeys.DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY);
}
-
- // copy over any/all YARN RM client values, in case the server-side XML conf file
- // has the 0.0.0.0 address
- commandLine.addConfOptions(config,
- YarnConfiguration.RM_ADDRESS,
- YarnConfiguration.RM_CLUSTER_ID,
- YarnConfiguration.RM_HOSTNAME,
- YarnConfiguration.RM_PRINCIPAL);
-
// write out the path output
commandLine.addOutAndErrFiles(STDOUT_AM, STDERR_AM);
diff --git a/slider-core/src/main/java/org/apache/slider/core/launch/JavaCommandLineBuilder.java b/slider-core/src/main/java/org/apache/slider/core/launch/JavaCommandLineBuilder.java
index dcb322e..0b3fa10 100644
--- a/slider-core/src/main/java/org/apache/slider/core/launch/JavaCommandLineBuilder.java
+++ b/slider-core/src/main/java/org/apache/slider/core/launch/JavaCommandLineBuilder.java
@@ -20,15 +20,11 @@
import com.google.common.base.Preconditions;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.api.ApplicationConstants;
import org.apache.slider.common.tools.SliderUtils;
-import org.apache.slider.core.exceptions.BadConfigException;
/**
- * Command line builder purely for the Java CLI.
- * Some of the <code>define</code> methods are designed to work with Hadoop tool and
- * Slider launcher applications.
+ * Command line builder purely for the Java CLI
*/
public class JavaCommandLineBuilder extends CommandLineBuilder {
@@ -84,68 +80,4 @@
sysprop("java.awt.headless", "true");
return this;
}
-
- public boolean addConfOption(Configuration conf, String key) {
- return defineIfSet(key, conf.get(key));
- }
-
- /**
- * Add a varargs list of configuration parameters —if they are present
- * @param conf configuration source
- * @param keys keys
- */
- public void addConfOptions(Configuration conf, String...keys) {
- for (String key : keys) {
- addConfOption(conf, key);
- }
- }
-
- public String addConfOptionToCLI(Configuration conf,
- String key,
- String defVal) {
- String val = conf.get(key, defVal);
- define(key, val);
- return val;
- }
-
- /**
- * Add a <code>-D key=val</code> command to the CLI. This is very Hadoop API
- * @param key key
- * @param val value
- */
- public void define(String key, String val) {
- Preconditions.checkArgument(key != null, "null key");
- Preconditions.checkArgument(val != null, "null value");
- add("-D", key + "=" + val);
- }
-
- /**
- * Add a <code>-D key=val</code> command to the CLI if <code>val</code>
- * is not null
- * @param key key
- * @param val value
- */
- public boolean defineIfSet(String key, String val) {
- Preconditions.checkArgument(key != null, "null key");
- if (val != null) {
- define(key, val);
- return true;
- } else {
- return false;
- }
- }
-
- /**
- * Add a mandatory config option
- * @param conf configuration
- * @param key key
- * @throws BadConfigException if the key is missing
- */
- public void addMandatoryConfOption(Configuration conf,
- String key) throws BadConfigException {
- if (!addConfOption(conf, key)) {
- throw new BadConfigException("Missing configuration option: " + key);
- }
- }
-
}
diff --git a/slider-core/src/main/java/org/apache/slider/providers/ProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/ProviderService.java
index 1a18db6..28cef01 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/ProviderService.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/ProviderService.java
@@ -186,10 +186,10 @@
/**
* Prior to going live -register the initial service registry data
- * @param amWebURI URL to the AM. This may be proxied, so use relative paths
- * @param agentOpsURI URI for agent operations. This will not be proxied
- * @param agentStatusURI URI For agent status. Again: no proxy
- * @param serviceRecord service record to build up
+ * @param amWebURI
+ * @param agentOpsURI
+ * @param agentStatusURI
+ * @param serviceRecord
*/
void applyInitialRegistryDefinitions(URL amWebURI,
URL agentOpsURI,
diff --git a/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMProviderService.java
index be33c21..cee7a97 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMProviderService.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMProviderService.java
@@ -142,10 +142,10 @@
try {
- URL managementAPI = new URL(amWebURI, RELATIVE_PATH_MANAGEMENT);
- URL registryREST = new URL(amWebURI, RELATIVE_PATH_REGISTRY);
+ URL managementAPI = new URL(amWebURI, SLIDER_PATH_MANAGEMENT);
+ URL registryREST = new URL(amWebURI, SLIDER_PATH_REGISTRY );
- URL publisherURL = new URL(amWebURI, RELATIVE_PATH_PUBLISHER);
+ URL publisherURL = new URL(amWebURI, SLIDER_PATH_PUBLISHER);
// Set the configurations URL.
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index 12709bc..45e2ff9 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -47,7 +47,6 @@
import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse;
import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
-import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
@@ -83,7 +82,6 @@
import org.apache.slider.api.RoleKeys;
import org.apache.slider.api.StatusKeys;
import org.apache.slider.api.proto.SliderClusterAPI;
-import org.apache.slider.client.SliderYarnClientImpl;
import org.apache.slider.common.SliderExitCodes;
import org.apache.slider.common.SliderKeys;
import org.apache.slider.common.params.AbstractActionArgs;
@@ -275,10 +273,6 @@
@SuppressWarnings("FieldAccessedSynchronizedAndUnsynchronized")
private String appMasterTrackingUrl = "";
- /** Proxied app master URL (as retrieved from AM report at launch time) */
- @SuppressWarnings("FieldAccessedSynchronizedAndUnsynchronized")
- private String appMasterProxiedUrl = "";
-
/** Application Attempt Id ( combination of attemptId and fail count )*/
private ApplicationAttemptId appAttemptID;
@@ -418,7 +412,6 @@
*/
private boolean securityEnabled;
private ContentCache contentCache;
- private SliderYarnClientImpl yarnClient;
/**
* Service Constructor
@@ -679,22 +672,10 @@
sliderAMProvider = new SliderAMProviderService();
initAndAddService(sliderAMProvider);
- InetSocketAddress rmSchedulerAddress = SliderUtils.getRmSchedulerAddress(serviceConf);
- log.info("RM is at {}", rmSchedulerAddress);
+ InetSocketAddress address = SliderUtils.getRmSchedulerAddress(serviceConf);
+ log.info("RM is at {}", address);
yarnRPC = YarnRPC.create(serviceConf);
- // set up the YARN client. This may require patching in the RM client-API address if it
- // is (somehow) unset server-side. String clientRMaddr = serviceConf.get(YarnConfiguration.RM_ADDRESS);
- InetSocketAddress clientRpcAddress = SliderUtils.getRmAddress(serviceConf);
- if (!SliderUtils.isAddressDefined(clientRpcAddress)) {
- // client addr is being unset. We can lift it from the other RM APIs
- log.warn("Yarn RM address was unbound; attempting to fix up");
- serviceConf.set(YarnConfiguration.RM_ADDRESS,
- String.format("%s:%d", rmSchedulerAddress.getHostString(), clientRpcAddress.getPort() ));
- }
- initAndAddService(yarnClient = new SliderYarnClientImpl());
- yarnClient.start();
-
/*
* Extract the container ID. This is then
* turned into an (incompete) container
@@ -849,17 +830,6 @@
}
}
- // look up the application itself -this is needed to get the proxied
- // URL of the AM, for registering endpoints.
- // this call must be made after the AM has registered itself, obviously
- ApplicationAttemptReport report = yarnClient.getApplicationAttemptReport(
- appAttemptID);
- appMasterProxiedUrl = report.getTrackingUrl();
- if (SliderUtils.isUnset(appMasterProxiedUrl)) {
- log.warn("Proxied URL is not set in application report");
- appMasterProxiedUrl = appMasterTrackingUrl;
- }
-
// extract container list
liveContainers = amRegistrationData.getContainersFromPreviousAttempts();
@@ -984,8 +954,7 @@
// log the YARN and web UIs
log.info("RM Webapp address {}",
serviceConf.get(YarnConfiguration.RM_WEBAPP_ADDRESS));
- log.info("Slider webapp address {} proxied at {}",
- appMasterTrackingUrl, appMasterProxiedUrl);
+ log.info("slider Webapp address {}", appMasterTrackingUrl);
// Start the Slider AM provider
sliderAMProvider.start();
@@ -1225,7 +1194,7 @@
// the registry is running, so register services
- URL amWebURI = new URL(appMasterProxiedUrl);
+ URL amWebURI = new URL(appMasterTrackingUrl);
URL agentOpsURI = new URL(agentOpsUrl);
URL agentStatusURI = new URL(agentStatusUrl);
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/RestPaths.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/RestPaths.java
index 17fe31f..06b7ba2 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/RestPaths.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/RestPaths.java
@@ -35,10 +35,8 @@
* agent content root: {@value}
*/
public static final String WS_AGENT_CONTEXT_ROOT = "/" + AGENT_WS_CONTEXT;
- public static final String V1_SLIDER = "/v1/slider";
- public static final String SLIDER_CONTEXT_ROOT = WS_CONTEXT_ROOT + V1_SLIDER;
- public static final String RELATIVE_API = WS_CONTEXT + V1_SLIDER;
- public static final String SLIDER_AGENT_CONTEXT_ROOT = WS_AGENT_CONTEXT_ROOT + V1_SLIDER;
+ public static final String SLIDER_CONTEXT_ROOT = WS_CONTEXT_ROOT +"/v1/slider";
+ public static final String SLIDER_AGENT_CONTEXT_ROOT = WS_AGENT_CONTEXT_ROOT +"/v1/slider";
public static final String MANAGEMENT = "mgmt";
public static final String SLIDER_SUBPATH_MANAGEMENT = "/" + MANAGEMENT;
public static final String SLIDER_SUBPATH_AGENTS = "/agents";
@@ -51,9 +49,6 @@
public static final String SLIDER_PATH_MANAGEMENT = SLIDER_CONTEXT_ROOT
+ SLIDER_SUBPATH_MANAGEMENT;
- public static final String RELATIVE_PATH_MANAGEMENT = RELATIVE_API
- + SLIDER_SUBPATH_MANAGEMENT;
-
/**
* Agents: {@value}
*/
@@ -66,9 +61,6 @@
public static final String SLIDER_PATH_PUBLISHER = SLIDER_CONTEXT_ROOT
+ SLIDER_SUBPATH_PUBLISHER;
- public static final String RELATIVE_PATH_PUBLISHER = RELATIVE_API
- + SLIDER_SUBPATH_PUBLISHER;
-
/**
* Registry subpath: {@value}
*/
@@ -79,8 +71,6 @@
*/
public static final String SLIDER_PATH_REGISTRY = SLIDER_CONTEXT_ROOT
+ SLIDER_SUBPATH_REGISTRY;
- public static final String RELATIVE_PATH_REGISTRY = RELATIVE_API
- + SLIDER_SUBPATH_REGISTRY;
/**
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneYarnRegistryAM.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneYarnRegistryAM.groovy
index 56fb5f2..139860a 100644
--- a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneYarnRegistryAM.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneYarnRegistryAM.groovy
@@ -18,13 +18,10 @@
package org.apache.slider.agent.standalone
-import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import org.apache.hadoop.fs.FileUtil
import org.apache.hadoop.fs.PathNotFoundException
import org.apache.hadoop.registry.client.binding.RegistryPathUtils
-import org.apache.hadoop.registry.client.types.AddressTypes
-import org.apache.hadoop.registry.client.types.ProtocolTypes
import org.apache.hadoop.yarn.api.records.ApplicationReport
import org.apache.hadoop.yarn.api.records.YarnApplicationState
import org.apache.hadoop.yarn.conf.YarnConfiguration
@@ -34,6 +31,7 @@
import org.apache.hadoop.registry.client.types.RegistryPathStatus
import org.apache.hadoop.registry.client.types.ServiceRecord
import org.apache.hadoop.registry.client.types.yarn.YarnRegistryAttributes
+import org.apache.slider.common.SliderExitCodes
import org.apache.slider.common.params.ActionResolveArgs
import org.apache.slider.common.params.Arguments
import org.apache.slider.core.exceptions.NotFoundException
@@ -61,7 +59,7 @@
/**
* work with a YARN registry
*/
-@CompileStatic
+//@CompileStatic
@Slf4j
class TestStandaloneYarnRegistryAM extends AgentMiniClusterTestBase {
@@ -167,19 +165,15 @@
assert serviceRecord[YarnRegistryAttributes.YARN_PERSISTENCE] != ""
def externalEndpoints = serviceRecord.external;
assert externalEndpoints.size() > 0
+/*
+
+ def am_ipc_protocol = AM_IPC_PROTOCOL
+ serviceRecord.getExternalEndpoint(am_ipc_protocol)
+ assert null != am_ipc_protocol;
+*/
assert null != serviceRecord.getExternalEndpoint(MANAGEMENT_REST_API)
assert null != serviceRecord.getExternalEndpoint(PUBLISHER_REST_API)
- def publisherEPR = serviceRecord.getExternalEndpoint(PUBLISHER_REST_API)
- assert publisherEPR.protocolType == ProtocolTypes.PROTOCOL_REST
- assert publisherEPR.addressType == AddressTypes.ADDRESS_URI
- def addr = publisherEPR.addresses[0][AddressTypes.ADDRESS_URI]
- def proxiedURL = new URL(addr)
- def appId = report.applicationId.toString()
- // something like http://host:49257/proxy/application_1449844996855_0001/ws/v1/slider/publisher
- String publisherPath = "/proxy/$appId/" + RestPaths.RELATIVE_PATH_PUBLISHER
- assert proxiedURL.path == publisherPath
-
// internals
assert null != serviceRecord.getInternalEndpoint(AGENT_ONEWAY_REST_API)
assert null != serviceRecord.getInternalEndpoint(AGENT_SECURE_REST_API)
@@ -329,8 +323,8 @@
String confJSON = GET(yarnSitePublisher)
// log.info(confJSON)
- JsonSerDeser<PublishedConfiguration> confSerDeser =
- new JsonSerDeser<>(PublishedConfiguration)
+ JsonSerDeser< PublishedConfiguration> confSerDeser =
+ new JsonSerDeser<PublishedConfiguration>(PublishedConfiguration)
publishedYarnSite = confSerDeser.fromJson(confJSON)
@@ -494,7 +488,7 @@
assert LauncherExitCodes.EXIT_NOT_FOUND == client.actionRegistry(registryArgs)
//look for a different user
- assertFailsWithException(LauncherExitCodes.EXIT_NOT_FOUND, "") {
+ assertFailsWithException(SliderExitCodes.EXIT_NOT_FOUND, "") {
def args = new ActionRegistryArgs(
name: clustername,
user: "unknown",
