SLIDER-5 CLI to list and fetch configurations - work in progress
git-svn-id: https://svn.apache.org/repos/asf/incubator/slider/trunk@1592520 13f79535-47bb-0310-9956-ffa450edef68
diff --git a/slider-core/pom.xml b/slider-core/pom.xml
index d0e881d..ca39be6 100644
--- a/slider-core/pom.xml
+++ b/slider-core/pom.xml
@@ -573,7 +573,6 @@
<dependency>
<groupId>com.sun.jersey.jersey-test-framework</groupId>
<artifactId>jersey-test-framework-grizzly2</artifactId>
- <version>${jersey.version}</version>
</dependency>
<dependency>
diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index b89077a..054fc66 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -84,7 +84,10 @@
import org.apache.slider.core.persist.ConfPersister;
import org.apache.slider.core.persist.LockAcquireFailedException;
import org.apache.slider.core.registry.YARNRegistryClient;
+import org.apache.slider.core.registry.docstore.PublishedConfigSet;
+import org.apache.slider.core.registry.docstore.PublishedConfiguration;
import org.apache.slider.core.registry.info.ServiceInstanceData;
+import org.apache.slider.core.registry.retrieve.RegistryRetriever;
import org.apache.slider.core.registry.zk.ZKPathBuilder;
import org.apache.slider.providers.AbstractClientProvider;
import org.apache.slider.providers.SliderProviderFactory;
@@ -94,7 +97,7 @@
import org.apache.slider.server.appmaster.rpc.RpcBinder;
import org.apache.slider.server.services.curator.CuratorServiceInstance;
import org.apache.slider.server.services.curator.RegistryBinderService;
-import org.apache.slider.server.services.docstore.utility.AbstractSliderLaunchedService;
+import org.apache.slider.server.services.utility.AbstractSliderLaunchedService;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -105,7 +108,6 @@
import java.io.StringWriter;
import java.io.Writer;
import java.net.InetSocketAddress;
-import java.util.Collection;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
@@ -117,9 +119,7 @@
*/
public class SliderClient extends AbstractSliderLaunchedService implements RunService,
- SliderExitCodes,
- SliderKeys,
- ErrorStrings {
+ SliderExitCodes, SliderKeys, ErrorStrings {
private static final Logger log = LoggerFactory.getLogger(SliderClient.class);
private ClientArgs serviceArgs;
@@ -1939,41 +1939,128 @@
/**
- * Status operation
+ * Registry operation
*
* @param registryArgs registry Arguments
- * @throws YarnException
- * @throws IOException
+ * @throws YarnException YARN problems
+ * @throws IOException Network or other problems
*/
@VisibleForTesting
public int actionRegistry(ActionRegistryArgs registryArgs) throws
YarnException,
IOException {
- maybeStartRegistry();
- List<CuratorServiceInstance<ServiceInstanceData>> instances =
- registry.listInstances(SliderKeys.APP_TYPE);
-
- for (CuratorServiceInstance<ServiceInstanceData> instance : instances) {
- log.info("{} at http://{}:{}/", instance.id, instance.address,
- instance.port);
+ // as this is also a test entry point, validate
+ // the arguments
+ registryArgs.validate();
+ int exitCode = EXIT_SUCCESS;
+ if (registryArgs.list) {
+ actionRegistryList(registryArgs);
+ } else if (registryArgs.listConf) {
+ // list the configurations
+ actionRegistryListConfigs(registryArgs);
+ } else {
+ exitCode = EXIT_FALSE;
}
- return EXIT_SUCCESS;
+ return exitCode;
}
/**
- * List names in the registry
- * @return
- * @throws IOException
- * @throws YarnException
+ * Registry operation
+ *
+ * @param registryArgs registry Arguments
+ * @throws YarnException YARN problems
+ * @throws IOException Network or other problems
*/
- public Collection<String> listRegistryNames() throws IOException, YarnException {
- Collection<String> names;
- verifyBindingsDefined();
+ private void actionRegistryList(ActionRegistryArgs registryArgs)
+ throws YarnException, IOException {
+ List<CuratorServiceInstance<ServiceInstanceData>> instances =
+ getRegistry().listInstances(registryArgs.serviceType);
- return getRegistry().queryForNames();
+ for (CuratorServiceInstance<ServiceInstanceData> instance : instances) {
+ if (!registryArgs.verbose) {
+ log.info("{}", instance.id);
+ } else {
+ log.info("{} ", instance);
+ }
+ }
}
/**
+ * Registry operation
+ *
+ * @param registryArgs registry Arguments
+ * @throws YarnException YARN problems
+ * @throws IOException Network or other problems
+ */
+ public void actionRegistryListConfigs(ActionRegistryArgs registryArgs)
+ throws YarnException, IOException {
+ ServiceInstanceData instance = lookupInstance(registryArgs);
+
+ RegistryRetriever retriever = new RegistryRetriever(instance);
+ PublishedConfigSet configurations =
+ retriever.getConfigurations(!registryArgs.internal);
+
+ for (String configName : configurations.keys()) {
+ if (!registryArgs.verbose) {
+ log.info("{}", configName);
+ } else {
+ PublishedConfiguration published =
+ configurations.get(configName);
+ log.info("{} : {}",
+ configName,
+ published.description);
+ }
+ }
+ }
+
+
+ /**
+ * Look up an instance
+ * @param id instance ID
+ * @param serviceType service type
+ * @return instance data
+ * @throws UnknownApplicationInstanceException no match
+ * @throws SliderException other failures
+ * @throws IOException IO problems or wrapped exceptions
+ */
+ private ServiceInstanceData lookupInstance(ActionRegistryArgs registryArgs) throws
+ UnknownApplicationInstanceException,
+ SliderException,
+ IOException {
+ return lookupInstance(registryArgs.name, registryArgs.serviceType);
+ }
+
+ /**
+ * Look up an instance
+ * @param id instance ID
+ * @param serviceType service type
+ * @return instance data
+ * @throws UnknownApplicationInstanceException no match
+ * @throws SliderException other failures
+ * @throws IOException IO problems or wrapped exceptions
+ */
+ private ServiceInstanceData lookupInstance(String id,
+ String serviceType) throws
+ UnknownApplicationInstanceException,
+ SliderException,
+ IOException {
+ try {
+ CuratorServiceInstance<ServiceInstanceData> csi =
+ getRegistry().queryForInstance(serviceType, id);
+ if (csi == null) {
+ throw new UnknownApplicationInstanceException(
+ "instance %s of type %s not found",
+ id, serviceType);
+ }
+ return csi.getPayload();
+ } catch (IOException e) {
+ throw e;
+ } catch (Exception e) {
+ throw new IOException(e);
+ }
+ }
+
+ /**
* List instances in the registry
* @return
* @throws IOException
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java b/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java
index 44fecea..1cec75e 100644
--- a/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java
+++ b/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java
@@ -48,6 +48,7 @@
String ARG_HELP = "--help";
String ARG_ID = "--id";
String ARG_IMAGE = "--image";
+ String ARG_INTERNAL = "--internal";
String ARG_LIST = "--list";
String ARG_LISTFILES = "--listfiles";
String ARG_LISTCONF = "--listconf";
@@ -57,7 +58,7 @@
String ARG_MESSAGE = "--message";
String ARG_OPTION = "--option";
String ARG_OPTION_SHORT = "-O";
- // String ARG_NAME = "--name";
+ String ARG_NAME = "--name";
String ARG_OUTPUT = "--out";
String ARG_OUTPUT_SHORT = "-o";
String ARG_PACKAGE = "--package";
@@ -68,8 +69,10 @@
String ARG_RESOURCE_MANAGER = "--rm";
String ARG_RESOURCE_OPT = "--resopt";
String ARG_RESOURCE_OPT_SHORT = "-ro";
+ String ARG_SERVICETYPE = "--servictype";
String ARG_SYSPROP = "-S";
String ARG_TEMPLATE = "--template";
+ String ARG_VERBOSE = "--verbose";
String ARG_WAIT = "--wait";
String ARG_ZKPATH = "--zkpath";
String ARG_ZKPORT = "--zkport";
diff --git a/slider-core/src/main/java/org/apache/slider/core/exceptions/ExceptionConverter.java b/slider-core/src/main/java/org/apache/slider/core/exceptions/ExceptionConverter.java
new file mode 100644
index 0000000..98dc028
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/core/exceptions/ExceptionConverter.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.core.exceptions;
+
+import com.sun.jersey.api.client.ClientResponse;
+import com.sun.jersey.api.client.UniformInterfaceException;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+
+/**
+ * static methods to convert exceptions into different types, including
+ * extraction of details and finer-grained conversions.
+ */
+public class ExceptionConverter {
+
+ /**
+ * Convert a Jersey Exception into an IOE or subclass
+ * @param targetURL URL being targeted
+ * @param exception original exception
+ * @return a new exception, the original one nested as a cause
+ */
+ public static IOException convertJerseyException(String targetURL,
+ UniformInterfaceException exception) {
+
+ ClientResponse response = exception.getResponse();
+ if (response != null) {
+ int status = response.getStatus();
+ if (status >= 400 && status < 500) {
+ FileNotFoundException fnfe =
+ new FileNotFoundException(targetURL);
+ fnfe.initCause(exception);
+ return fnfe;
+ }
+ }
+
+ return new IOException("Failed to GET " + targetURL + ": " + exception, exception);
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/core/registry/retrieve/RegistryRetriever.java b/slider-core/src/main/java/org/apache/slider/core/registry/retrieve/RegistryRetriever.java
new file mode 100644
index 0000000..1202cb8
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/core/registry/retrieve/RegistryRetriever.java
@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.core.registry.retrieve;
+
+import com.beust.jcommander.Strings;
+import com.sun.jersey.api.client.Client;
+import com.sun.jersey.api.client.UniformInterfaceException;
+import com.sun.jersey.api.client.WebResource;
+import org.apache.slider.core.exceptions.ExceptionConverter;
+import org.apache.slider.core.registry.docstore.PublishedConfigSet;
+import org.apache.slider.core.registry.info.RegistryView;
+import org.apache.slider.core.registry.info.ServiceInstanceData;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.ws.rs.core.MediaType;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+
+/**
+ * Registry retriever.
+ * This hides the HTTP operations that take place to
+ * get the actual content
+ */
+public class RegistryRetriever {
+ private static final Logger log = LoggerFactory.getLogger(RegistryRetriever.class);
+
+ private final ServiceInstanceData instance;
+ private static final Client jerseyClient;
+
+ static {
+ jerseyClient = Client.create();
+ jerseyClient.setFollowRedirects(true);
+ }
+
+
+ public RegistryRetriever(ServiceInstanceData instance) {
+ this.instance = instance;
+ }
+
+ /**
+ * Get the appropriate view for the flag
+ * @param external
+ * @return
+ */
+ private RegistryView getRegistryView(boolean external) {
+ return external ? instance.externalView : instance.internalView;
+ }
+
+ private String destination(boolean external) {
+ return external ? "external" : "internal";
+ }
+
+ /**
+ * Does a bonded registry retriever have a configuration?
+ * @param external flag to indicate that it is the external entries to fetch
+ * @return true if there is a URL to the configurations defined
+ */
+ public boolean hasConfigurations(boolean external) {
+ String confURL = getRegistryView(external).configurationsURL;
+ return !Strings.isStringEmpty(confURL);
+ }
+
+ /**
+ * Get the configurations of the registry
+ * @param external flag to indicate that it is the external entries to fetch
+ * @return the configuration sets
+ */
+ public PublishedConfigSet getConfigurations(boolean external) throws
+ FileNotFoundException, IOException {
+
+ String confURL = getRegistryView(external).configurationsURL;
+ if (Strings.isStringEmpty(confURL)) {
+ throw new FileNotFoundException("No configuration URL at "
+ + destination(external) + " view");
+ }
+ WebResource webResource = jerseyClient.resource(confURL);
+ try {
+ PublishedConfigSet configSet =
+ webResource.type(MediaType.APPLICATION_JSON)
+ .get(PublishedConfigSet.class);
+ return configSet;
+ } catch (UniformInterfaceException e) {
+ throw ExceptionConverter.convertJerseyException(confURL, e);
+ }
+ }
+
+ @Override
+ public String toString() {
+ return super.toString() + " - " + instance;
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index 7b4ed06..5eaa618 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -111,10 +111,10 @@
import org.apache.slider.server.services.curator.RegistryBinderService;
import org.apache.slider.server.services.curator.RegistryConsts;
import org.apache.slider.server.services.curator.RegistryNaming;
-import org.apache.slider.server.services.docstore.utility.AbstractSliderLaunchedService;
-import org.apache.slider.server.services.docstore.utility.EventCallback;
-import org.apache.slider.server.services.docstore.utility.RpcService;
-import org.apache.slider.server.services.docstore.utility.WebAppService;
+import org.apache.slider.server.services.utility.AbstractSliderLaunchedService;
+import org.apache.slider.server.services.utility.EventCallback;
+import org.apache.slider.server.services.utility.RpcService;
+import org.apache.slider.server.services.utility.WebAppService;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneRegistryAM.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneRegistryAM.groovy
index 7a4500a..0069e73 100644
--- a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneRegistryAM.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneRegistryAM.groovy
@@ -26,11 +26,13 @@
import org.apache.slider.api.ClusterNode
import org.apache.slider.client.SliderClient
import org.apache.slider.common.SliderKeys
+import org.apache.slider.common.params.ActionRegistryArgs
import org.apache.slider.core.main.ServiceLauncher
import org.apache.slider.core.persist.JsonSerDeser
import org.apache.slider.core.registry.docstore.PublishedConfigSet
import org.apache.slider.core.registry.info.CustomRegistryConstants
import org.apache.slider.core.registry.info.ServiceInstanceData
+import org.apache.slider.core.registry.retrieve.RegistryRetriever
import org.apache.slider.server.appmaster.web.rest.RestPaths
import org.apache.slider.server.services.curator.CuratorServiceInstance
import org.apache.slider.server.services.curator.RegistryBinderService
@@ -163,8 +165,42 @@
log.info(GET(registryURL, RestPaths.REGISTRY_SERVICE ))
+ describe "Registry Retrieval"
+ // retrieval
- describe "teardown of cluster"
+ RegistryRetriever retriever = new RegistryRetriever(serviceInstanceData)
+ log.info retriever.toString()
+
+ assert retriever.hasConfigurations(true)
+ def externalConf = retriever.getConfigurations(true)
+ externalConf.keys().each { String key ->
+ def config = externalConf.get(key)
+ log.info "$key -- ${config.description} -- size ${config.size}"
+ }
+
+ describe "Internal configurations"
+ assert !retriever.hasConfigurations(false)
+ try {
+ retriever.getConfigurations(false)
+ fail( "expected a failure")
+ } catch (FileNotFoundException fnfe) {
+ //expected
+ }
+
+ // retrieval via API
+ ActionRegistryArgs registryArgs = new ActionRegistryArgs()
+ registryArgs.name = amInstance;
+ registryArgs.verbose = true
+ registryArgs.list = true;
+ assert client.actionRegistry(registryArgs)
+
+ registryArgs.list = false;
+ registryArgs.listConf = true
+ assert client.actionRegistry(registryArgs)
+
+
+
+ describe "freeze cluster"
//now kill that cluster
assert 0 == clusterActionFreeze(client, clustername)
//list it & See if it is still there
diff --git a/slider-core/src/test/groovy/org/apache/slider/registry/curator/TestLocalRegistry.groovy b/slider-core/src/test/groovy/org/apache/slider/registry/curator/TestLocalRegistry.groovy
index 3af4a06..cf8dbd9 100644
--- a/slider-core/src/test/groovy/org/apache/slider/registry/curator/TestLocalRegistry.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/registry/curator/TestLocalRegistry.groovy
@@ -21,6 +21,7 @@
import org.apache.hadoop.yarn.conf.YarnConfiguration
import org.apache.slider.common.SliderKeys
import org.apache.slider.core.registry.info.ServiceInstanceData
+import org.apache.slider.core.registry.retrieve.RegistryRetriever
import org.apache.slider.server.services.curator.CuratorHelper
import org.apache.slider.server.services.curator.RegistryBinderService
import org.apache.slider.server.services.curator.RegistryNaming
@@ -154,6 +155,9 @@
SliderTestUtils.dumpRegistryInstances(instances)
assert instances.size() == 2
+ // now set up a registry retriever
+ RegistryRetriever retriever = new RegistryRetriever()
+
}
}
diff --git a/slider-core/src/test/groovy/org/apache/slider/registry/curator/TestRegistryRestResources.groovy b/slider-core/src/test/groovy/org/apache/slider/registry/curator/TestRegistryRestResources.groovy
index 1a7ee5a..d631a62 100644
--- a/slider-core/src/test/groovy/org/apache/slider/registry/curator/TestRegistryRestResources.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/registry/curator/TestRegistryRestResources.groovy
@@ -49,8 +49,9 @@
class TestRegistryRestResources extends AgentTestBase {
public static final String REGISTRY_URI = RestPaths.SLIDER_PATH_REGISTRY;
+ public static final String WADL = "vnd.sun.wadl+xml"
-
+
private String id(String instanceName) {
RegistryNaming.createUniqueInstanceId(
@@ -120,7 +121,7 @@
ClientResponse response = webResource.type(MediaType.APPLICATION_XML)
.get(ClientResponse.class);
assert response.status == 200
- assert response.getType() == (new MediaType("application", "vnd.sun.wadl+xml"))
+ assert response.type == (new MediaType("application", WADL))
// test the available GET URIs
webResource = client.resource(
diff --git a/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy b/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
index a64370d..9ccae32 100644
--- a/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
@@ -35,6 +35,7 @@
import org.apache.slider.client.SliderClient
import org.apache.slider.common.params.Arguments
import org.apache.slider.common.tools.Duration
+import org.apache.slider.common.tools.SliderUtils
import org.apache.slider.core.conf.AggregateConf
import org.apache.slider.core.exceptions.BadClusterStateException
import org.apache.slider.core.exceptions.SliderException
@@ -362,19 +363,8 @@
return fetchWebPageWithoutError(s)
}
- def static String appendToURL(String base, String path) {
- StringBuilder fullpath = new StringBuilder(base)
- if (!base.endsWith("/")) {
- fullpath.append("/")
- }
- if (path.startsWith("/")) {
- fullpath.append(path.substring(1))
- } else {
- fullpath.append(path)
- }
-
- def s = fullpath.toString()
- return s
+ public static String appendToURL(String base, String path) {
+ return SliderUtils.appendToURL(base, path)
}
/**
