SLIDER-62 add functional test for registry entries against hbase provider
git-svn-id: https://svn.apache.org/repos/asf/incubator/slider/trunk@1595221 13f79535-47bb-0310-9956-ffa450edef68
diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index bd9ff7c..225cce5 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -1971,6 +1971,8 @@
registryArgs);
}
} catch (FileNotFoundException e) {
+ log.info("{}", e.toString());
+ log.debug("{}", e);
return EXIT_NOT_FOUND;
}
return EXIT_SUCCESS;
@@ -2079,7 +2081,14 @@
external);
return published;
}
-
+
+ /**
+ * write out the config
+ * @param published
+ * @param registryArgs
+ * @throws BadCommandArgumentsException
+ * @throws IOException
+ */
private void outputConfig(PublishedConfiguration published,
ActionRegistryArgs registryArgs) throws
BadCommandArgumentsException,
@@ -2136,16 +2145,14 @@
*/
private ServiceInstanceData lookupInstance(String id,
String serviceType) throws
- UnknownApplicationInstanceException,
- SliderException,
IOException {
try {
CuratorServiceInstance<ServiceInstanceData> csi =
getRegistry().queryForInstance(serviceType, id);
if (csi == null) {
- throw new UnknownApplicationInstanceException(
- "instance %s of type %s not found",
- id, serviceType);
+ throw new FileNotFoundException(
+ String.format("instance %s of type %s not found",
+ id, serviceType));
}
return csi.getPayload();
} catch (IOException e) {
diff --git a/slider-core/src/main/java/org/apache/slider/common/SliderXMLConfKeysForTesting.java b/slider-core/src/main/java/org/apache/slider/common/SliderXMLConfKeysForTesting.java
index 6d5a328..6d3fe38 100644
--- a/slider-core/src/main/java/org/apache/slider/common/SliderXMLConfKeysForTesting.java
+++ b/slider-core/src/main/java/org/apache/slider/common/SliderXMLConfKeysForTesting.java
@@ -64,4 +64,11 @@
String KEY_TEST_TEARDOWN_KILLALL = "slider.test.teardown.killall";
boolean DEFAULT_TEARDOWN_KILLALL = true;
+
+
+ /**
+ * Key for amount of RAM to request
+ */
+ String KEY_TEST_YARN_RAM_REQUEST = "slider.test.yarn.ram";
+ String DEFAULT_YARN_RAM_REQUEST = "192";
}
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/ActionRegistryArgs.java b/slider-core/src/main/java/org/apache/slider/common/params/ActionRegistryArgs.java
index b041051..9d311c2 100644
--- a/slider-core/src/main/java/org/apache/slider/common/params/ActionRegistryArgs.java
+++ b/slider-core/src/main/java/org/apache/slider/common/params/ActionRegistryArgs.java
@@ -23,6 +23,7 @@
import org.apache.slider.common.SliderKeys;
import org.apache.slider.core.exceptions.BadCommandArgumentsException;
import org.apache.slider.core.exceptions.ErrorStrings;
+import org.apache.slider.core.registry.docstore.ConfigFormat;
import static org.apache.slider.common.params.SliderActions.ACTION_REGISTRY;
import static org.apache.slider.common.params.SliderActions.DESCRIBE_ACTION_REGISTRY;
@@ -41,6 +42,14 @@
commandDescription = DESCRIBE_ACTION_REGISTRY)
public class ActionRegistryArgs extends AbstractActionArgs {
+
+ public ActionRegistryArgs() {
+ }
+
+ public ActionRegistryArgs(String name) {
+ this.name = name;
+ }
+
@Override
public String getActionName() {
return ACTION_REGISTRY;
@@ -81,7 +90,7 @@
//--format
@Parameter(names = ARG_FORMAT,
description = "Format for a response: [xml|json|properties]")
- public String format;
+ public String format = ConfigFormat.XML.toString() ;
@Parameter(names = {ARG_DEST},
@@ -124,11 +133,6 @@
throw new BadCommandArgumentsException("Argument " + ARG_DEST
+ " is only supported on 'get' operations");
}
- if (is(format) && !is(getConf)) {
- throw new BadCommandArgumentsException("Argument " + ARG_FORMAT
- + " is only supported by " + ARG_GETCONF);
-
- }
if (!list && !is(name)) {
throw new BadCommandArgumentsException("Argument " + ARG_NAME
+" missing");
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneRegistryAM.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneRegistryAM.groovy
index 182939a..b8eecf1 100644
--- a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneRegistryAM.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneRegistryAM.groovy
@@ -306,27 +306,32 @@
registryArgs.list = false;
registryArgs.listConf = false
registryArgs.internal = false
- registryArgs.format = "properties"
def yarn_site_config = PublishedArtifacts.YARN_SITE_CONFIG
registryArgs.getConf = yarn_site_config
-
-
+
+ //properties format
+ registryArgs.format = "properties"
describe registryArgs.toString()
+
client.actionRegistry(registryArgs)
+
File outputDir = new File("target/test_standalone_registry_am/output")
outputDir.mkdirs()
+ // create a new registry args with the defaults back in
+ registryArgs = new ActionRegistryArgs(serviceInstanceData.id)
+ registryArgs.getConf = yarn_site_config
registryArgs.dest = outputDir
describe registryArgs.toString()
client.actionRegistry(registryArgs)
- assert new File(outputDir, yarn_site_config + ".properties").exists()
-
- registryArgs.format = "xml"
- client.actionRegistry(registryArgs)
assert new File(outputDir, yarn_site_config + ".xml").exists()
+ registryArgs.format = "properties"
+ client.actionRegistry(registryArgs)
+ assert new File(outputDir, yarn_site_config + ".properties").exists()
+
describe registryArgs.toString()
def unknownFilename = "undefined-file"
diff --git a/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy b/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
index a784a0a..c6e1f0e 100644
--- a/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
@@ -32,6 +32,7 @@
import org.apache.hadoop.yarn.conf.YarnConfiguration
import org.apache.slider.api.ClusterDescription
import org.apache.slider.api.ClusterNode
+import org.apache.slider.api.RoleKeys
import org.apache.slider.client.SliderClient
import org.apache.slider.common.params.Arguments
import org.apache.slider.common.tools.Duration
@@ -259,7 +260,15 @@
if (instanceCount != desiredCount) {
roleCountFound = false;
}
- details.append("[$role]: desired: $desiredCount; actual: $instanceCount ")
+ details.append("[$role]: desired: $desiredCount;" +
+ " actual: $instanceCount ")
+
+ // call out requested count, as this is a cause of problems on
+ // overloaded functional test clusters
+ def requested = status.roles[role][RoleKeys.ROLE_REQUESTED_INSTANCES]
+ if (requested != "0") {
+ details.append("requested: $requested ")
+ }
}
if (roleCountFound) {
//successful
@@ -277,8 +286,9 @@
duration.finish();
describe("$operation: role count not met after $duration: $details")
log.info(prettyPrint(status.toJsonString()))
- fail(
- "$operation: role counts not met after $duration: $details in \n$status ")
+ fail("$operation: role counts not met after $duration: " +
+ details.toString() +
+ " in \n$status ")
}
log.debug("Waiting: " + details)
Thread.sleep(1000)
diff --git a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
index b5b685c..c69b8ab 100644
--- a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
+++ b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
@@ -49,7 +49,7 @@
abstract class CommandTestBase extends SliderTestUtils {
private static final Logger log =
LoggerFactory.getLogger(CommandTestBase.class);
-
+
public static final String SLIDER_CONF_DIR = sysprop(SLIDER_CONF_DIR_PROP)
public static final String SLIDER_BIN_DIR = sysprop(SLIDER_BIN_DIR_PROP)
public static final File SLIDER_BIN_DIRECTORY = new File(
@@ -57,7 +57,8 @@
public static final File SLIDER_SCRIPT = new File(
SLIDER_BIN_DIRECTORY,
BIN_SLIDER).canonicalFile
- public static final File SLIDER_CONF_DIRECTORY = new File(SLIDER_CONF_DIR).canonicalFile
+ public static final File SLIDER_CONF_DIRECTORY = new File(
+ SLIDER_CONF_DIR).canonicalFile
public static final File SLIDER_CONF_XML = new File(SLIDER_CONF_DIRECTORY,
CLIENT_CONFIG_FILENAME).canonicalFile
@@ -65,16 +66,17 @@
public static final int THAW_WAIT_TIME
public static final int FREEZE_WAIT_TIME
- public static final int ACCUMULO_LAUNCH_WAIT_TIME
public static final int SLIDER_TEST_TIMEOUT
- public static final boolean ACCUMULO_TESTS_ENABLED
public static final boolean FUNTESTS_ENABLED
+ public static final String YARN_RAM_REQUEST
+
+
static {
- SLIDER_CONFIG = new ConfLoader().loadSliderConf(SLIDER_CONF_XML);
- THAW_WAIT_TIME = getTimeOptionMillis(SLIDER_CONFIG,
+ SLIDER_CONFIG = new ConfLoader().loadSliderConf(SLIDER_CONF_XML);
+ THAW_WAIT_TIME = getTimeOptionMillis(SLIDER_CONFIG,
KEY_TEST_THAW_WAIT_TIME,
1000 * DEFAULT_THAW_WAIT_TIME_SECONDS)
FREEZE_WAIT_TIME = getTimeOptionMillis(SLIDER_CONFIG,
@@ -83,15 +85,14 @@
SLIDER_TEST_TIMEOUT = getTimeOptionMillis(SLIDER_CONFIG,
KEY_TEST_TIMEOUT,
1000 * DEFAULT_TEST_TIMEOUT_SECONDS)
- ACCUMULO_LAUNCH_WAIT_TIME = getTimeOptionMillis(SLIDER_CONFIG,
- KEY_ACCUMULO_LAUNCH_TIME,
- 1000 * DEFAULT_ACCUMULO_LAUNCH_TIME_SECONDS)
FUNTESTS_ENABLED =
SLIDER_CONFIG.getBoolean(KEY_SLIDER_FUNTESTS_ENABLED, true)
- ACCUMULO_TESTS_ENABLED =
- SLIDER_CONFIG.getBoolean(KEY_TEST_ACCUMULO_ENABLED, false)
- }
+ YARN_RAM_REQUEST = SLIDER_CONFIG.get(
+ KEY_TEST_YARN_RAM_REQUEST,
+ DEFAULT_YARN_RAM_REQUEST)
+
+ }
@Rule
public final Timeout testTimeout = new Timeout(SLIDER_TEST_TIMEOUT);
@@ -110,9 +111,9 @@
SliderShell.script = SLIDER_SCRIPT
log.info("Test using ${HadoopFS.getDefaultUri(SLIDER_CONFIG)} " +
"and YARN RM @ ${SLIDER_CONFIG.get(YarnConfiguration.RM_ADDRESS)}")
-
+
// now patch the settings with the path of the conf direcotry
-
+
}
/**
@@ -143,7 +144,7 @@
}
return property
}
-
+
/**
* Exec any slider command
* @param conf
@@ -217,7 +218,7 @@
ACTION_FREEZE, name
])
}
-
+
static SliderShell freeze(String name, Collection<String> args) {
slider([ACTION_FREEZE, name] + args)
}
@@ -234,18 +235,18 @@
static SliderShell getConf(int result, String name) {
slider(result,
- [
- ACTION_GETCONF, name
- ])
+ [
+ ACTION_GETCONF, name
+ ])
}
-
+
static SliderShell killContainer(String name, String containerID) {
slider(0,
- [
- ACTION_KILL_CONTAINER,
- name,
- containerID
- ])
+ [
+ ACTION_KILL_CONTAINER,
+ name,
+ containerID
+ ])
}
static SliderShell list(String name) {
@@ -276,9 +277,9 @@
static SliderShell status(int result, String name) {
slider(result,
- [
- ACTION_STATUS, name
- ])
+ [
+ ACTION_STATUS, name
+ ])
}
static SliderShell thaw(String name) {
@@ -289,23 +290,24 @@
static SliderShell thaw(int result, String name) {
slider(result,
- [
- ACTION_THAW, name
- ])
+ [
+ ACTION_THAW, name
+ ])
}
static SliderShell thaw(String name, Collection<String> args) {
slider([ACTION_THAW, name] + args)
}
-
+
static SliderShell registry(int result, Collection<String> commands) {
slider(result,
- [ ACTION_REGISTRY ] + commands
+ [ACTION_REGISTRY] + commands
)
}
+
static SliderShell registry(Collection<String> commands) {
slider(0,
- [ ACTION_REGISTRY ] + commands
+ [ACTION_REGISTRY] + commands
)
}
@@ -325,7 +327,7 @@
/**
* If the functional tests are enabled, set up the cluster
- *
+ *
* @param cluster
*/
static void setupCluster(String cluster) {
@@ -419,8 +421,8 @@
List<String> argsList = [action, clustername]
argsList << ARG_ZKHOSTS <<
- SLIDER_CONFIG.getTrimmed(SliderXmlConfKeys.REGISTRY_ZK_QUORUM)
-
+ SLIDER_CONFIG.getTrimmed(SliderXmlConfKeys.REGISTRY_ZK_QUORUM)
+
if (blockUntilRunning) {
argsList << ARG_WAIT << Integer.toString(THAW_WAIT_TIME)
@@ -470,13 +472,15 @@
}
public Path buildClusterPath(String clustername) {
- return new Path(clusterFS.homeDirectory, "${SliderKeys.SLIDER_BASE_DIRECTORY}/cluster/${clustername}")
+ return new Path(
+ clusterFS.homeDirectory,
+ "${SliderKeys.SLIDER_BASE_DIRECTORY}/cluster/${clustername}")
}
public ClusterDescription killAmAndWaitForRestart(
SliderClient sliderClient, String cluster) {
-
+
assert cluster
slider(0, [
ACTION_AM_SUICIDE, cluster,
@@ -487,8 +491,8 @@
- def sleeptime = SLIDER_CONFIG.getInt( KEY_AM_RESTART_SLEEP_TIME,
- DEFAULT_AM_RESTART_SLEEP_TIME)
+ def sleeptime = SLIDER_CONFIG.getInt(KEY_AM_RESTART_SLEEP_TIME,
+ DEFAULT_AM_RESTART_SLEEP_TIME)
sleep(sleeptime)
ClusterDescription status
@@ -515,9 +519,4 @@
assume(FUNTESTS_ENABLED, "Functional tests disabled")
}
- public static void assumeAccumuloTestsEnabled() {
- assumeFunctionalTestsEnabled()
- assume(ACCUMULO_TESTS_ENABLED, "Accumulo tests disabled")
- }
-
}
diff --git a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/SliderShell.groovy b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/SliderShell.groovy
index 841e19e..068c330 100644
--- a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/SliderShell.groovy
+++ b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/SliderShell.groovy
@@ -146,7 +146,10 @@
assert shell != null
if (shell.ret != errorCode) {
shell.dump()
- throw new SliderException(shell.ret,"Expected exit code %d - actual=%d", errorCode, shell.ret)
+ throw new SliderException(shell.ret,
+ "Expected exit code of command %s : %d - actual=%d",
+ shell.command,
+ errorCode, shell.ret)
}
return errorCode
}
diff --git a/slider-providers/accumulo/accumulo-funtests/src/test/groovy/org/apache/slider/providers/accumulo/funtest/AccumuloCommandTestBase.groovy b/slider-providers/accumulo/accumulo-funtests/src/test/groovy/org/apache/slider/providers/accumulo/funtest/AccumuloCommandTestBase.groovy
index 1a97635..1b5d8bf 100644
--- a/slider-providers/accumulo/accumulo-funtests/src/test/groovy/org/apache/slider/providers/accumulo/funtest/AccumuloCommandTestBase.groovy
+++ b/slider-providers/accumulo/accumulo-funtests/src/test/groovy/org/apache/slider/providers/accumulo/funtest/AccumuloCommandTestBase.groovy
@@ -21,6 +21,9 @@
import static SliderXMLConfKeysForTesting.KEY_TEST_ACCUMULO_APPCONF
import static SliderXMLConfKeysForTesting.KEY_TEST_ACCUMULO_TAR
import static org.apache.slider.api.ResourceKeys.YARN_MEMORY
+import static org.apache.slider.common.SliderXMLConfKeysForTesting.DEFAULT_ACCUMULO_LAUNCH_TIME_SECONDS
+import static org.apache.slider.common.SliderXMLConfKeysForTesting.KEY_ACCUMULO_LAUNCH_TIME
+import static org.apache.slider.common.SliderXMLConfKeysForTesting.KEY_TEST_ACCUMULO_ENABLED
import static org.apache.slider.providers.accumulo.AccumuloKeys.*
import static org.apache.slider.common.params.Arguments.ARG_PROVIDER
import static org.apache.slider.common.params.Arguments.ARG_RES_COMP_OPT
@@ -46,6 +49,23 @@
*/
abstract class AccumuloCommandTestBase extends CommandTestBase {
+ public static final int ACCUMULO_LAUNCH_WAIT_TIME
+ public static final boolean ACCUMULO_TESTS_ENABLED
+
+ static {
+ ACCUMULO_LAUNCH_WAIT_TIME = getTimeOptionMillis(SLIDER_CONFIG,
+ KEY_ACCUMULO_LAUNCH_TIME,
+ 1000 * DEFAULT_ACCUMULO_LAUNCH_TIME_SECONDS)
+ ACCUMULO_TESTS_ENABLED =
+ SLIDER_CONFIG.getBoolean(KEY_TEST_ACCUMULO_ENABLED, false)
+ }
+
+
+ public static void assumeAccumuloTestsEnabled() {
+ assumeFunctionalTestsEnabled()
+ assume(ACCUMULO_TESTS_ENABLED, "Accumulo tests disabled")
+ }
+
@Before
public void verifyPreconditions() {
diff --git a/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/HBaseCommandTestBase.groovy b/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/HBaseCommandTestBase.groovy
index 6e1f91b..fe9370e 100644
--- a/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/HBaseCommandTestBase.groovy
+++ b/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/HBaseCommandTestBase.groovy
@@ -18,16 +18,20 @@
package org.apache.slider.providers.hbase.funtest
+import org.apache.slider.api.ResourceKeys
+import org.apache.slider.common.params.Arguments
import org.apache.slider.funtest.categories.FunctionalTests
import org.apache.slider.funtest.framework.CommandTestBase
import org.apache.slider.funtest.framework.SliderShell
-import org.apache.slider.common.params.Arguments
import org.apache.slider.providers.hbase.HBaseClientProvider
import org.apache.slider.providers.hbase.HBaseKeys
import org.junit.Before
import org.junit.BeforeClass
import static org.apache.slider.common.SliderXMLConfKeysForTesting.*
+import static org.apache.slider.common.params.Arguments.ARG_RES_COMP_OPT
+import static org.apache.slider.providers.hbase.HBaseKeys.ROLE_MASTER
+import static org.apache.slider.providers.hbase.HBaseKeys.ROLE_WORKER
/**
* Anything specific to HBase tests
@@ -41,9 +45,9 @@
static {
HBASE_TESTS_ENABLED =
SLIDER_CONFIG.getBoolean(KEY_TEST_HBASE_ENABLED, true)
- HBASE_LAUNCH_WAIT_TIME = SLIDER_CONFIG.getInt(
- KEY_TEST_HBASE_LAUNCH_TIME,
- DEFAULT_HBASE_LAUNCH_TIME_SECONDS)
+ HBASE_LAUNCH_WAIT_TIME = getTimeOptionMillis(SLIDER_CONFIG,
+ KEY_TEST_HBASE_LAUNCH_TIME,
+ 1000 * DEFAULT_HBASE_LAUNCH_TIME_SECONDS)
}
@BeforeClass
@@ -92,6 +96,11 @@
SLIDER_CONFIG.getTrimmed(KEY_TEST_HBASE_APPCONF)
argsList << Arguments.ARG_PROVIDER << HBaseKeys.PROVIDER_HBASE
+ // add the RAM requirements
+ argsList << ARG_RES_COMP_OPT << ROLE_MASTER <<
+ ResourceKeys.YARN_MEMORY << YARN_RAM_REQUEST
+ argsList << ARG_RES_COMP_OPT << ROLE_WORKER <<
+ ResourceKeys.YARN_MEMORY << YARN_RAM_REQUEST
SliderShell shell = createSliderApplication(
name,
diff --git a/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/TestFunctionalHBaseCluster.groovy b/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/TestFunctionalHBaseCluster.groovy
index 4c5a1b1..eff0555 100644
--- a/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/TestFunctionalHBaseCluster.groovy
+++ b/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/TestFunctionalHBaseCluster.groovy
@@ -21,20 +21,25 @@
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import org.apache.hadoop.conf.Configuration
+import org.apache.hadoop.yarn.conf.YarnConfiguration
+import org.apache.slider.api.ClusterDescription
+import org.apache.slider.api.RoleKeys
+import org.apache.slider.client.SliderClient
import org.apache.slider.common.SliderExitCodes
import org.apache.slider.common.SliderKeys
import org.apache.slider.common.SliderXmlConfKeys
-import org.apache.slider.api.ClusterDescription
-import org.apache.slider.api.RoleKeys
+import org.apache.slider.common.params.Arguments
+import org.apache.slider.common.tools.ConfigHelper
import org.apache.slider.core.registry.info.RegistryNaming
import org.apache.slider.funtest.framework.FuntestProperties
-import org.apache.slider.common.tools.ConfigHelper
-import org.apache.slider.common.params.Arguments
-import org.apache.slider.client.SliderClient
import org.apache.slider.providers.hbase.HBaseConfigFileOptions
import org.apache.slider.providers.hbase.HBaseTestUtils
import org.apache.slider.server.appmaster.PublishedArtifacts
-import org.apache.zookeeper.*
+import org.apache.zookeeper.KeeperException
+import org.apache.zookeeper.WatchedEvent
+import org.apache.zookeeper.Watcher
+import org.apache.zookeeper.ZKUtil
+import org.apache.zookeeper.ZooKeeper
import org.junit.After
import org.junit.Before
import org.junit.Test
@@ -120,8 +125,10 @@
Configuration clientConf = HBaseTestUtils.createHBaseConfiguration(sliderClient)
HBaseTestUtils.assertHBaseMasterFound(clientConf)
- HBaseTestUtils.waitForHBaseRegionServerCount(sliderClient, clusterName,
- numWorkers, HBASE_LAUNCH_WAIT_TIME)
+ HBaseTestUtils.waitForHBaseRegionServerCount(sliderClient,
+ clusterName,
+ numWorkers,
+ HBASE_LAUNCH_WAIT_TIME)
clusterOperations(
clusterName,
@@ -172,8 +179,6 @@
//unknown service type
registry(EXIT_NOT_FOUND,
[ARG_LIST, ARG_SERVICETYPE, "org.apache.something"])
- registry(EXIT_NOT_FOUND,
- [ARG_LIST, ARG_SERVICETYPE, ""])
registry(EXIT_NOT_FOUND,
[ARG_LIST, ARG_NAME, "cluster-with-no-name"])
@@ -190,16 +195,29 @@
registry(EXIT_NOT_FOUND, [ARG_LISTCONF, ARG_NAME, "unknown"])
registry([ARG_GETCONF, PublishedArtifacts.COMPLETE_CONFIG,
ARG_NAME, name])
- registry([ARG_GETCONF, "no-such-config",
+ registry(EXIT_NOT_FOUND, [ARG_GETCONF, "no-such-config",
ARG_NAME, name])
- registry([ARG_GETCONF, "illegal config name!",
+ registry(EXIT_NOT_FOUND, [ARG_GETCONF, "illegal/config/name!",
ARG_NAME, name])
registry(EXIT_NOT_FOUND,[ARG_GETCONF, PublishedArtifacts.COMPLETE_CONFIG,
ARG_NAME, name, ARG_INTERNAL])
+ def yarn_site_config = PublishedArtifacts.YARN_SITE_CONFIG
+ registry([ARG_GETCONF, yarn_site_config,
+ ARG_NAME, name])
+
+ File getConfDir = new File("target/$clusterName/getconf")
+ getConfDir.mkdirs();
+ registry([ARG_GETCONF, yarn_site_config,
+ ARG_NAME, name,
+ ARG_DEST, getConfDir.absolutePath])
+ File retrieved = new File(getConfDir, yarn_site_config +".xml")
+ def confFromFile = ConfigHelper.loadConfFromFile(retrieved)
+ assert confFromFile.get(YarnConfiguration.RM_ADDRESS)
+
}
}
diff --git a/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/TestHBaseClusterBuildDestroy.groovy b/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/TestHBaseClusterBuildDestroy.groovy
index 4114a29..3c9b8ed 100644
--- a/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/TestHBaseClusterBuildDestroy.groovy
+++ b/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/TestHBaseClusterBuildDestroy.groovy
@@ -21,13 +21,12 @@
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import org.apache.hadoop.fs.Path
-import org.apache.slider.core.main.LauncherExitCodes
import org.apache.slider.common.SliderKeys
import org.apache.slider.common.SliderXmlConfKeys
-import org.apache.slider.funtest.framework.CommandTestBase
-import org.apache.slider.funtest.framework.FuntestProperties
import org.apache.slider.common.params.Arguments
import org.apache.slider.common.params.SliderActions
+import org.apache.slider.core.main.LauncherExitCodes
+import org.apache.slider.funtest.framework.FuntestProperties
import org.apache.slider.providers.hbase.HBaseKeys
import org.junit.AfterClass
import org.junit.BeforeClass
diff --git a/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/TestHBaseClusterLifecycle.groovy b/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/TestHBaseClusterLifecycle.groovy
index 22f246c..01c7131 100644
--- a/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/TestHBaseClusterLifecycle.groovy
+++ b/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/TestHBaseClusterLifecycle.groovy
@@ -20,14 +20,14 @@
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
-import org.apache.slider.common.SliderExitCodes
-import org.apache.slider.common.SliderXmlConfKeys
import org.apache.slider.api.ClusterDescription
import org.apache.slider.api.StatusKeys
-import org.apache.slider.funtest.framework.FuntestProperties
+import org.apache.slider.client.SliderClient
+import org.apache.slider.common.SliderExitCodes
+import org.apache.slider.common.SliderXmlConfKeys
import org.apache.slider.common.params.Arguments
import org.apache.slider.common.params.SliderActions
-import org.apache.slider.client.SliderClient
+import org.apache.slider.funtest.framework.FuntestProperties
import org.junit.After
import org.junit.Before
import org.junit.Test
diff --git a/src/site/markdown/developing/functional_tests.md b/src/site/markdown/developing/functional_tests.md
index d4e931e..d58c0ea 100644
--- a/src/site/markdown/developing/functional_tests.md
+++ b/src/site/markdown/developing/functional_tests.md
@@ -24,7 +24,7 @@
# Functional Tests
-The functional test suite is designed to run the executables against
+The functional test suite is designed to test slider against
a live cluster.
For these to work you need
@@ -112,6 +112,12 @@
<value>180000</value>
</property>
+ <property>
+ <name>slider.test.yarn.ram</name>
+ <description>Size in MB to ask for containers</description>
+ <value>192</value>
+ </property>
+
Note that while the same properties need to be set in
`slider-core/src/test/resources/slider-client.xml`, those tests take a file in the local
@@ -215,7 +221,7 @@
<property>
<name>slider.test.hbase.launch.wait.seconds</name>
<description>Time to wait in seconds for HBase to start</description>
- <value>180000</value>
+ <value>1800</value>
</property>
#### Accumulo configuration options
@@ -233,7 +239,7 @@
<property>
<name>slider.test.accumulo.launch.wait.seconds</name>
<description>Time to wait in seconds for Accumulo to start</description>
- <value>180000</value>
+ <value>1800</value>
</property>
### Configuring the YARN cluster for tests
@@ -360,9 +366,9 @@
mvn install -DskipTests
1. Start the YARN cluster/set up proxies to connect to it, etc.
-1. In the `slider-funtest` dir, run the test
+1. In the `slider-funtest` dir, run the tests
- mvn test -Dtest=TestHBaseCreateCluster
+ mvn test
A common mistake during development is to rebuild the `slider-core` JARs
then the `slider-funtest` tests without rebuilding the `slider-assembly`.
@@ -383,6 +389,10 @@
## Troubleshooting the functional tests
+1. If application instances fail to come up as there are still outstanding
+requests, it means that YARN didn't have the RAM/cores to spare for the number
+of containers. Edit the `slider.test.yarn.ram` to make it smaller.
+
1. If you are testing in a local VM and stops responding, it'll have been
swapped out to RAM. Rebooting can help, but for a long term fix go through
all the Hadoop configurations (HDFS, YARN, Zookeeper) and set their heaps to
diff --git a/src/site/markdown/manpage.md b/src/site/markdown/manpage.md
index 6dc94cb..e701db2 100644
--- a/src/site/markdown/manpage.md
+++ b/src/site/markdown/manpage.md
@@ -387,7 +387,7 @@
#### `registry --getconf <configuration> [--format (xml|json|properties)] [--dest <path>] [--internal] ` get the configuration
-Get a named configuration
+Get a named configuration in a chosen format. Default: XML
`--dest <path>` : the filename or directory to save a configuration to.
`--format (xml|json|properties)` defines the output format
diff --git a/src/test/clusters/sandbox/slider/slider-client.xml b/src/test/clusters/sandbox/slider/slider-client.xml
index 2dd3f51..770ab9b 100644
--- a/src/test/clusters/sandbox/slider/slider-client.xml
+++ b/src/test/clusters/sandbox/slider/slider-client.xml
@@ -31,7 +31,7 @@
<property>
<name>fs.defaultFS</name>
- <value>hdfs://sandbox:8020</value>
+ <value>hdfs://sandbox.hortonworks.com:8020</value>
</property>
<property>
@@ -67,7 +67,7 @@
<property>
<name>slider.test.hbase.tar</name>
<description>Path to the HBase Tar file in HDFS</description>
- <value>hdfs://sandbox:8020/hbase-0.98.1-bin.tar.gz</value>
+ <value>${fs.defaultFS}/hbase-0.98.1-bin.tar.gz</value>
</property>
@@ -88,7 +88,7 @@
<property>
<name>slider.test.accumulo.tar</name>
<description>Path to the Accumulo Tar file in HDFS</description>
- <value>hdfs://sandbox:8020/user/slider/accumulo.tar</value>
+ <value>${fs.defaultFS}/user/slider/accumulo.tar</value>
</property>
