SLIDER-5 CLI to list and retrieve service bindings & configs
this patch sets up the docs and tests ready for a deployment
git-svn-id: https://svn.apache.org/repos/asf/incubator/slider/trunk@1593056 13f79535-47bb-0310-9956-ffa450edef68
diff --git a/pom.xml b/pom.xml
index 4366643..b10680d 100644
--- a/pom.xml
+++ b/pom.xml
@@ -149,7 +149,7 @@
-->
<project.java.src.version>1.7</project.java.src.version>
<enforced.java.version>${project.java.src.version}</enforced.java.version>
-
+ <groovy.version>2.2.2</groovy.version>
<!--
options
@@ -180,7 +180,6 @@
<commons-lang.version>2.6</commons-lang.version>
<curator.version>2.4.1</curator.version>
<easymock.version>3.1</easymock.version>
- <groovy.version>2.1.9</groovy.version>
<guava.version>11.0.2</guava.version>
<gson.version>2.2.2</gson.version>
<guice.version>3.0</guice.version>
@@ -208,9 +207,9 @@
<!-- Plugin versions -->
<gmavenVersion>1.5</gmavenVersion>
<gmavenProviderSelection>2.0</gmavenProviderSelection>
- <buildnumber-maven-plugin.version>1.2</buildnumber-maven-plugin.version>
<groovy-eclipse-compiler.version>2.8.0-01</groovy-eclipse-compiler.version>
- <groovy-eclipse-batch.version>2.1.3-01</groovy-eclipse-batch.version>
+ <groovy-eclipse-batch.version>2.1.8-01</groovy-eclipse-batch.version>
+ <buildnumber-maven-plugin.version>1.2</buildnumber-maven-plugin.version>
<maven.version.range>[3.0.0,)</maven.version.range>
diff --git a/slider-agent/pom.xml b/slider-agent/pom.xml
index 6cf4082..f2fdae1 100644
--- a/slider-agent/pom.xml
+++ b/slider-agent/pom.xml
@@ -37,6 +37,7 @@
<build>
<plugins>
+
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-assembly-plugin</artifactId>
@@ -54,6 +55,7 @@
</execution>
</executions>
</plugin>
+
<plugin>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.0</version>
@@ -83,6 +85,7 @@
</execution>
</executions>
</plugin>
+
<plugin>
<groupId>org.apache.rat</groupId>
<artifactId>apache-rat-plugin</artifactId>
diff --git a/slider-assembly/pom.xml b/slider-assembly/pom.xml
index fb7603b..605f377 100644
--- a/slider-assembly/pom.xml
+++ b/slider-assembly/pom.xml
@@ -61,6 +61,7 @@
<artifactId>maven-assembly-plugin</artifactId>
<version>${maven-assembly-plugin.version}</version>
<configuration>
+ <tarLongFileMode>gnu</tarLongFileMode>
<descriptors>
<descriptor>src/assembly/slider-bin.xml</descriptor>
</descriptors>
@@ -100,6 +101,33 @@
</executions>
</plugin>
+ <!-- copy in the agent tar file -->
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-dependency-plugin</artifactId>
+ <version>${maven-dependency-plugin.version}</version>
+ <executions>
+ <execution>
+ <id>copy</id>
+ <phase>package</phase>
+ <goals>
+ <goal>copy</goal>
+ </goals>
+ <configuration>
+ <artifactItems>
+ <artifactItem>
+ <groupId>org.apache.slider</groupId>
+ <artifactId>slider-agent</artifactId>
+ <type>tar.gz</type>
+ <overWrite>false</overWrite>
+ <outputDirectory>${project.build.directory}/agent</outputDirectory>
+ <destFileName>slider-agent.tar.gz</destFileName>
+ </artifactItem>
+ </artifactItems>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
@@ -177,6 +205,18 @@
<version>${project.version}</version>
</dependency>
+ <!--
+ needed to order the build and ensure the agent tar is found
+ the test scope ensures that it isn't copied into the lib dir
+ -->
+ <dependency>
+ <groupId>org.apache.slider</groupId>
+ <artifactId>slider-agent</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ <type>tar.gz</type>
+ </dependency>
+
<dependency>
<groupId>com.beust</groupId>
<artifactId>jcommander</artifactId>
@@ -188,13 +228,7 @@
<artifactId>hadoop-client</artifactId>
<type>pom</type>
</dependency>
-
- <!--
- <dependency>
- <groupId>org.apache.zookeeper</groupId>
- <artifactId>zookeeper</artifactId>
- </dependency>
- -->
+
</dependencies>
diff --git a/slider-assembly/src/assembly/slider-bin.xml b/slider-assembly/src/assembly/slider-bin.xml
index 0c371a3..6f3b021 100644
--- a/slider-assembly/src/assembly/slider-bin.xml
+++ b/slider-assembly/src/assembly/slider-bin.xml
@@ -58,10 +58,10 @@
</fileSet>
<fileSet>
- <directory>${project.build.directory}/../../slider-agent/target</directory>
+ <directory>${project.build.directory}/agent</directory>
<outputDirectory>agent</outputDirectory>
<includes>
- <include>*.tar.gz</include>
+ <include>slider-agent.tar.gz</include>
</includes>
</fileSet>
diff --git a/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java b/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java
index a108092..b14610f 100644
--- a/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java
+++ b/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java
@@ -501,8 +501,8 @@
* @throws SliderException if the path does not exist
*/
public Path createPathThatMustExist(String uri) throws
- SliderException,
- IOException {
+ SliderException, IOException {
+ Preconditions.checkNotNull(uri);
Path path = new Path(uri);
verifyPathExists(path);
return path;
diff --git a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
index fae3e3f..3785484 100644
--- a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
+++ b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
@@ -1275,11 +1275,10 @@
}
public static Path extractImagePath(CoreFileSystem fs, MapOperations internalOptions) throws
- SliderException,
- IOException {
+ SliderException, IOException {
Path imagePath;
String imagePathOption =
- internalOptions.get(OptionKeys.INTERNAL_APPLICATION_IMAGE_PATH);
+ internalOptions.get(OptionKeys.INTERNAL_APPLICATION_IMAGE_PATH);
String appHomeOption = internalOptions.get(OptionKeys.INTERNAL_APPLICATION_HOME);
if (!isUnset(imagePathOption)) {
imagePath = fs.createPathThatMustExist(imagePathOption);
diff --git a/slider-core/src/test/groovy/org/apache/slider/common/tools/TestZKIntegration.groovy b/slider-core/src/test/groovy/org/apache/slider/common/tools/TestZKIntegration.groovy
index 3770656..0b4c66f 100644
--- a/slider-core/src/test/groovy/org/apache/slider/common/tools/TestZKIntegration.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/common/tools/TestZKIntegration.groovy
@@ -31,7 +31,6 @@
import org.junit.Test
@Slf4j
-@CompileStatic
class TestZKIntegration extends YarnZKMiniClusterTestBase implements KeysForTests {
diff --git a/slider-funtest/pom.xml b/slider-funtest/pom.xml
index e720100..71898b3 100644
--- a/slider-funtest/pom.xml
+++ b/slider-funtest/pom.xml
@@ -249,8 +249,16 @@
<groupId>org.apache.bigtop.itest</groupId>
<artifactId>itest-common</artifactId>
</dependency>
-
+ <dependency>
+ <groupId>org.apache.slider</groupId>
+ <artifactId>slider-agent</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ <type>tar.gz</type>
+ </dependency>
+
+
</dependencies>
diff --git a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
index d6b077e..dac8c8c 100644
--- a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
+++ b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
@@ -56,7 +56,7 @@
SLIDER_BIN_DIR).canonicalFile
public static final File SLIDER_SCRIPT = new File(
SLIDER_BIN_DIRECTORY,
- "bin/slider").canonicalFile
+ BIN_SLIDER).canonicalFile
public static final File SLIDER_CONF_DIRECTORY = new File(SLIDER_CONF_DIR).canonicalFile
public static final File SLIDER_CONF_XML = new File(SLIDER_CONF_DIRECTORY,
CLIENT_CONFIG_FILENAME).canonicalFile
@@ -70,8 +70,6 @@
public static final boolean ACCUMULO_TESTS_ENABLED
public static final boolean FUNTESTS_ENABLED
- public static final boolean AGENTTESTS_ENABLED
- public static final String AGENT_PKG
static {
@@ -93,10 +91,6 @@
ACCUMULO_TESTS_ENABLED =
SLIDER_CONFIG.getBoolean(KEY_TEST_ACCUMULO_ENABLED, false)
- AGENTTESTS_ENABLED =
- SLIDER_CONFIG.getBoolean(KEY_TEST_AGENT_ENABLED, true)
- AGENT_PKG =
- SLIDER_CONFIG.getRaw(KEY_TEST_AGENT_TAR)
}
@Rule
@@ -405,14 +399,6 @@
assert action != null
assert clustername != null
-
-
- List<String> roleList = [];
- roles.each { String role, Integer val ->
- log.info("Role $role := $val")
- roleList << ARG_COMPONENT << role << Integer.toString(val)
- }
-
List<String> argsList = [action, clustername]
argsList << ARG_ZKHOSTS <<
@@ -423,6 +409,12 @@
argsList << ARG_WAIT << Integer.toString(THAW_WAIT_TIME)
}
+ List<String> roleList = [];
+ roles.each { String role, Integer val ->
+ log.info("Role $role := $val")
+ roleList << ARG_COMPONENT << role << Integer.toString(val)
+ }
+
argsList += roleList;
//now inject any cluster options
@@ -511,9 +503,4 @@
assume(ACCUMULO_TESTS_ENABLED, "Accumulo tests disabled")
}
- public void assumeAgentTestsEnabled() {
- assumeFunctionalTestsEnabled()
- assume(AGENTTESTS_ENABLED, "Agent tests disabled")
- }
-
}
diff --git a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FuntestProperties.groovy b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FuntestProperties.groovy
index 7157b37..636da7b 100644
--- a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FuntestProperties.groovy
+++ b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FuntestProperties.groovy
@@ -60,4 +60,6 @@
String SCRIPT_NAME = "slider"
static final String KEY_TEST_CONF_XML = "slider.test.conf.xml"
static final String KEY_TEST_CONF_DIR = "slider.test.conf.dir"
+ static final String BIN_SLIDER = "bin/slider"
+ static final String AGENT_SLIDER_GZ = "agent/slider-agent.tar.gz"
}
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentCommandTestBase.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentCommandTestBase.groovy
new file mode 100644
index 0000000..36fab27
--- /dev/null
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentCommandTestBase.groovy
@@ -0,0 +1,113 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.funtest.lifecycle
+
+import groovy.util.logging.Slf4j
+import org.apache.hadoop.fs.Path
+import org.apache.slider.common.SliderExitCodes
+import org.apache.slider.common.params.Arguments
+import org.apache.slider.common.params.SliderActions
+import org.apache.slider.funtest.framework.CommandTestBase
+import org.apache.slider.funtest.framework.FuntestProperties
+import org.apache.slider.funtest.framework.SliderShell
+import org.junit.Before
+import org.junit.BeforeClass
+
+
+@Slf4j
+class AgentCommandTestBase extends CommandTestBase
+ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
+
+ public static final boolean AGENTTESTS_ENABLED
+
+ protected static String APP_RESOURCE = "../slider-core/src/test/app_packages/test_command_log/resources.json"
+ protected static String APP_TEMPLATE = "../slider-core/src/test/app_packages/test_command_log/appConfig.json"
+ protected static final File LOCAL_SLIDER_AGENT_TARGZ
+
+ protected static Path agentTarballPath;
+
+ static {
+ AGENTTESTS_ENABLED = SLIDER_CONFIG.getBoolean(KEY_TEST_AGENT_ENABLED, false)
+ LOCAL_SLIDER_AGENT_TARGZ = new File(
+ SLIDER_BIN_DIRECTORY,
+ AGENT_SLIDER_GZ).canonicalFile
+ }
+
+ @BeforeClass
+ public static void setupAgent() {
+ assumeAgentTestsEnabled()
+ assume(LOCAL_SLIDER_AGENT_TARGZ.exists(), "Slider agent not found at $LOCAL_SLIDER_AGENT_TARGZ")
+ agentTarballPath = new Path(clusterFS.homeDirectory, "agent.tar.gz")
+
+ Path localTarball = new Path(LOCAL_SLIDER_AGENT_TARGZ.toURI());
+ clusterFS.copyFromLocalFile(false, true, localTarball, agentTarballPath)
+ }
+
+ public static void assumeAgentTestsEnabled() {
+ assumeFunctionalTestsEnabled()
+ assume(AGENTTESTS_ENABLED, "Agent tests disabled")
+ }
+
+ public static void logShell(SliderShell shell) {
+ for (String str in shell.out) {
+ log.info str
+ }
+ }
+
+ public static void assertComponentCount(String component, int count, SliderShell shell) {
+ log.info("Asserting component count.")
+ String entry = findLineEntry(shell, ["instances", component] as String[])
+ log.info(entry)
+ assert entry != null
+ int instanceCount = 0
+ int index = entry.indexOf("container_")
+ while (index != -1) {
+ instanceCount++;
+ index = entry.indexOf("container_", index + 1)
+ }
+
+ assert instanceCount == count, 'Instance count for component did not match expected. Parsed: ' + entry
+ }
+
+ public static String findLineEntry(SliderShell shell, String[] locators) {
+ int index = 0;
+ for (String str in shell.out) {
+ if (str.contains("\"" + locators[index] + "\"")) {
+ if (locators.size() == index + 1) {
+ return str;
+ } else {
+ index++;
+ }
+ }
+ }
+
+ return null;
+ }
+
+ public static boolean isAppRunning(String text, SliderShell shell) {
+ boolean exists = false
+ for (String str in shell.out) {
+ if (str.contains(text)) {
+ exists = true
+ }
+ }
+
+ return exists
+ }
+}
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/TestAppsThroughAgent.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/TestAppsThroughAgent.groovy
index 34c3a57..694641e 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/TestAppsThroughAgent.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/TestAppsThroughAgent.groovy
@@ -20,26 +20,21 @@
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
-import org.apache.slider.common.params.Arguments
-import org.apache.slider.common.params.SliderActions
-import org.apache.slider.funtest.framework.CommandTestBase
import org.apache.slider.funtest.framework.SliderShell
import org.junit.Test
-import static org.apache.slider.common.SliderExitCodes.EXIT_UNKNOWN_INSTANCE
-
@CompileStatic
@Slf4j
-public class TestAppsThroughAgent extends CommandTestBase {
+public class TestAppsThroughAgent extends AgentCommandTestBase
+// implements FuntestProperties, Arguments, SliderExitCodes, SliderActions
+{
private static String COMMAND_LOGGER = "COMMAND_LOGGER"
private static String APPLICATION_NAME = "agenttst"
- private static String APP_RESOURCE = "../slider-core/src/test/app_packages/test_command_log/resources.json"
- private static String APP_TEMPLATE = "../slider-core/src/test/app_packages/test_command_log/appConfig.json"
@Test
public void testUsage() throws Throwable {
- SliderShell shell = slider(0, [SliderActions.ACTION_USAGE])
+ SliderShell shell = slider(EXIT_SUCCESS, [ACTION_USAGE])
assertSuccess(shell)
}
@@ -52,25 +47,21 @@
cleanup()
try {
- SliderShell shell = slider(0, [
- SliderActions.ACTION_CREATE,
- TestAppsThroughAgent.APPLICATION_NAME,
- Arguments.ARG_IMAGE,
- AGENT_PKG,
- Arguments.ARG_TEMPLATE,
- TestAppsThroughAgent.APP_TEMPLATE,
- Arguments.ARG_RESOURCES,
- TestAppsThroughAgent.APP_RESOURCE])
+ SliderShell shell = slider(EXIT_SUCCESS,
+ [
+ ACTION_CREATE, APPLICATION_NAME,
+ ARG_IMAGE, agentTarballPath.toString(),
+ ARG_TEMPLATE, APP_TEMPLATE,
+ ARG_RESOURCES, APP_RESOURCE
+ ])
logShell(shell)
- assertSuccess(shell)
-
int attemptCount = 0
while (attemptCount < 10) {
- shell = slider(0, [
- SliderActions.ACTION_LIST,
- TestAppsThroughAgent.APPLICATION_NAME])
+ shell = slider(EXIT_SUCCESS, [
+ ACTION_LIST,
+ APPLICATION_NAME])
if (isAppRunning("RUNNING", shell)) {
break
@@ -83,25 +74,28 @@
}
//flex
- slider(0, [
- SliderActions.ACTION_FLEX,
- TestAppsThroughAgent.APPLICATION_NAME,
- Arguments.ARG_COMPONENT,
+ slider(EXIT_SUCCESS,
+ [
+ ACTION_FLEX,
+ APPLICATION_NAME,
+ ARG_COMPONENT,
COMMAND_LOGGER,
"2"])
// sleep till the new instance starts
sleep(1000 * 10)
- shell = slider(0, [
- SliderActions.ACTION_STATUS,
- TestAppsThroughAgent.APPLICATION_NAME])
+ shell = slider(EXIT_SUCCESS,
+ [
+ ACTION_STATUS,
+ APPLICATION_NAME])
assertComponentCount(COMMAND_LOGGER, 2, shell)
- shell = slider(0, [
- SliderActions.ACTION_LIST,
- TestAppsThroughAgent.APPLICATION_NAME])
+ shell = slider(EXIT_SUCCESS,
+ [
+ ACTION_LIST,
+ APPLICATION_NAME])
assert isAppRunning("RUNNING", shell), 'App is not running.'
@@ -111,59 +105,12 @@
}
}
- public void logShell(SliderShell shell) {
- for (String str in shell.out) {
- log.info str
- }
- }
-
-
- public void assertComponentCount(String component, int count, SliderShell shell) {
- log.info("Asserting component count.")
- String entry = findLineEntry(shell, ["instances", component] as String[])
- log.info(entry)
- assert entry != null
- int instanceCount = 0
- int index = entry.indexOf("container_")
- while (index != -1) {
- instanceCount++;
- index = entry.indexOf("container_", index + 1)
- }
-
- assert instanceCount == count, 'Instance count for component did not match expected. Parsed: ' + entry
- }
-
- public String findLineEntry(SliderShell shell, String[] locators) {
- int index = 0;
- for (String str in shell.out) {
- if (str.contains("\"" + locators[index] + "\"")) {
- if (locators.size() == index + 1) {
- return str;
- } else {
- index++;
- }
- }
- }
-
- return null;
- }
-
- public boolean isAppRunning(String text, SliderShell shell) {
- boolean exists = false
- for (String str in shell.out) {
- if (str.contains(text)) {
- exists = true
- }
- }
-
- return exists
- }
public void cleanup() throws Throwable {
- log.info "Cleaning app instance, if exists, by name " + TestAppsThroughAgent.APPLICATION_NAME
+ log.info "Cleaning app instance, if exists, by name " + APPLICATION_NAME
SliderShell shell = slider([
- SliderActions.ACTION_FREEZE,
- TestAppsThroughAgent.APPLICATION_NAME])
+ ACTION_FREEZE,
+ APPLICATION_NAME])
if (shell.ret != 0 && shell.ret != EXIT_UNKNOWN_INSTANCE) {
logShell(shell)
@@ -174,8 +121,8 @@
sleep(1000 * 5)
shell = slider([
- SliderActions.ACTION_DESTROY,
- TestAppsThroughAgent.APPLICATION_NAME])
+ ACTION_DESTROY,
+ APPLICATION_NAME])
if (shell.ret != 0 && shell.ret != EXIT_UNKNOWN_INSTANCE) {
logShell(shell)
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/TestClusterBuildDestroy.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/TestClusterBuildDestroy.groovy
new file mode 100644
index 0000000..5c1743e
--- /dev/null
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/TestClusterBuildDestroy.groovy
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.funtest.lifecycle
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.hadoop.fs.Path
+import org.apache.slider.common.SliderExitCodes
+import org.apache.slider.common.SliderKeys
+import org.apache.slider.common.SliderXmlConfKeys
+import org.apache.slider.common.params.Arguments
+import org.apache.slider.common.params.SliderActions
+import org.apache.slider.funtest.framework.FuntestProperties
+import org.junit.AfterClass
+import org.junit.BeforeClass
+import org.junit.Test
+
+@CompileStatic
+@Slf4j
+public class TestClusterBuildDestroy extends AgentCommandTestBase {
+
+
+ static String CLUSTER = "test_cluster_build_destroy"
+
+
+ @BeforeClass
+ public static void prepareCluster() {
+ assumeFunctionalTestsEnabled();
+ setupCluster(CLUSTER)
+ }
+
+ @AfterClass
+ public static void destroyCluster() {
+ teardown(CLUSTER)
+ }
+
+ @Test
+ public void testBuildAndDestroyCluster() throws Throwable {
+ def clusterDir = SliderKeys.SLIDER_BASE_DIRECTORY + "/cluster/$CLUSTER"
+ def clusterDirPath = new Path(clusterFS.homeDirectory, clusterDir)
+ clusterFS.delete(clusterDirPath, true)
+ slider(EXIT_SUCCESS,
+ [
+ ACTION_BUILD,
+ CLUSTER,
+ ARG_ZKHOSTS,
+ SLIDER_CONFIG.get(SliderXmlConfKeys.REGISTRY_ZK_QUORUM, DEFAULT_SLIDER_ZK_HOSTS),
+ ])
+
+
+ assert clusterFS.exists(clusterDirPath)
+ //cluster exists if you don't want it to be live
+ exists(EXIT_SUCCESS, CLUSTER, false)
+ // condition returns false if it is required to be live
+ exists(EXIT_FALSE, CLUSTER, true)
+ destroy(CLUSTER)
+
+ }
+
+
+}
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/TestClusterLifecycle.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/TestClusterLifecycle.groovy
new file mode 100644
index 0000000..45e1db0
--- /dev/null
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/TestClusterLifecycle.groovy
@@ -0,0 +1,189 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.funtest.lifecycle
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.slider.api.ClusterDescription
+import org.apache.slider.api.StatusKeys
+import org.apache.slider.client.SliderClient
+import org.apache.slider.common.SliderExitCodes
+import org.apache.slider.common.SliderXmlConfKeys
+import org.apache.slider.common.params.Arguments
+import org.apache.slider.common.params.SliderActions
+import org.apache.slider.funtest.framework.FuntestProperties
+import org.junit.After
+import org.junit.Before
+import org.junit.Test
+
+@CompileStatic
+@Slf4j
+public class TestClusterLifecycle extends AgentCommandTestBase
+ implements FuntestProperties, Arguments, SliderExitCodes {
+
+
+ static String CLUSTER = "test_cluster_lifecycle"
+
+
+ @Before
+ public void prepareCluster() {
+ setupCluster(CLUSTER)
+ }
+
+ @After
+ public void destroyCluster() {
+ teardown(CLUSTER)
+ }
+
+ @Test
+ public void testClusterLifecycle() throws Throwable {
+
+ describe "Walk a 0-role cluster through its lifecycle"
+
+
+ def clusterpath = buildClusterPath(CLUSTER)
+ assert !clusterFS.exists(clusterpath)
+
+/*
+
+ Map<String, Integer> roleMap = createHBaseCluster(CLUSTER,
+ 0,
+ 0,
+ [],
+ [:])
+
+*/
+
+ //at this point the cluster should exist.
+ assertPathExists(clusterFS,"Cluster parent directory does not exist", clusterpath.parent)
+
+ assertPathExists(clusterFS,"Cluster directory does not exist", clusterpath)
+
+ // assert it exists on the command line
+ exists(0, CLUSTER)
+
+ //destroy will fail in use
+
+ destroy(EXIT_APPLICATION_IN_USE, CLUSTER)
+
+ //thaw will fail as cluster is in use
+ thaw(EXIT_APPLICATION_IN_USE, CLUSTER)
+
+ //it's still there
+ exists(0, CLUSTER)
+
+ //listing the cluster will succeed
+ list(0, CLUSTER)
+
+ //simple status
+ status(0, CLUSTER)
+
+ //now status to a temp file
+ File jsonStatus = File.createTempFile("tempfile", ".json")
+ try {
+ slider(0,
+ [
+ SliderActions.ACTION_STATUS, CLUSTER,
+ ARG_OUTPUT, jsonStatus.canonicalPath
+ ])
+
+ assert jsonStatus.exists()
+ ClusterDescription cd = ClusterDescription.fromFile(jsonStatus)
+
+ assert CLUSTER == cd.name
+
+ log.info(cd.toJsonString())
+
+ getConf(0, CLUSTER)
+
+ //get a slider client against the cluster
+ SliderClient sliderClient = bondToCluster(SLIDER_CONFIG, CLUSTER)
+ ClusterDescription cd2 = sliderClient.getClusterDescription()
+ assert CLUSTER == cd2.name
+
+ log.info("Connected via Client {}", sliderClient.toString())
+
+ //freeze
+ slider(0, [
+ SliderActions.ACTION_FREEZE, CLUSTER,
+ ARG_WAIT, Integer.toString(FREEZE_WAIT_TIME),
+ ARG_MESSAGE, "freeze-in-testHBaseCreateCluster"
+ ])
+
+ //cluster exists if you don't want it to be live
+ exists(0, CLUSTER, false)
+ // condition returns false if it is required to be live
+ exists(EXIT_FALSE, CLUSTER, true)
+
+
+ // thaw then freeze the cluster
+
+ slider(0,
+ [
+ SliderActions.ACTION_THAW, CLUSTER,
+ ARG_WAIT, Integer.toString(THAW_WAIT_TIME),
+ ])
+ exists(0, CLUSTER)
+ slider(0, [
+ SliderActions.ACTION_FREEZE, CLUSTER,
+ ARG_FORCE,
+ ARG_WAIT, Integer.toString(FREEZE_WAIT_TIME),
+ ARG_MESSAGE, "forced-freeze-in-test"
+ ])
+
+ //cluster is no longer live
+ exists(0, CLUSTER, false)
+
+ // condition returns false if it is required to be live
+ exists(EXIT_FALSE, CLUSTER, true)
+
+ // thaw with a restart count set to enable restart
+
+ describe "the kill/restart phase may fail if yarn.resourcemanager.am.max-attempts is too low"
+ slider(0,
+ [
+ SliderActions.ACTION_THAW, CLUSTER,
+ ARG_WAIT, Integer.toString(THAW_WAIT_TIME),
+ ARG_DEFINE, SliderXmlConfKeys.KEY_AM_RESTART_LIMIT + "=3"
+ ])
+
+
+
+ ClusterDescription status = killAmAndWaitForRestart(sliderClient, CLUSTER)
+
+ def restarted = status.getInfo(
+ StatusKeys.INFO_CONTAINERS_AM_RESTART)
+ assert restarted != null
+ assert Integer.parseInt(restarted) == 0
+ freeze(CLUSTER)
+
+ destroy(0, CLUSTER)
+
+ //cluster now missing
+ exists(EXIT_UNKNOWN_INSTANCE, CLUSTER)
+
+ } finally {
+ jsonStatus.delete()
+ }
+
+
+ }
+
+
+}
diff --git a/slider-funtest/src/test/manual/testing.md b/slider-funtest/src/test/manual/testing.md
deleted file mode 100644
index 018d785..0000000
--- a/slider-funtest/src/test/manual/testing.md
+++ /dev/null
@@ -1,51 +0,0 @@
-<!---
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-# Manual Testing
-
-Manual testing invloves using Slider package and an AppPackage to perform basic cluster functionalities such as create/destroy, flex up/down, and freeze/thaw. A python helper script is provided that can be used to automatically test and app package.
-
-## SliderTester.py
-Details to be added.
-
-## SliderTester.ini
-The various config parameters are:
-
-* **slider**
- * *package:* location of the slider package
- * *jdk.path:* jdk path on the test hosts
-
-* **app**
- * *package:* location of the app package
-
-* **cluster**
- * *yarn.application.classpath:* yarn application classpaths
- * *slider.zookeeper.quorum:* the ZK quorum hosts
- * *yarn.resourcemanager.address:*
- * *yarn.resourcemanager.scheduler.address:*
- * *fs.defaultFS:* e.g. hdfs://NN_HOST:8020
-
-* **test**
- * *app.user:* user to use for app creation
- * *hdfs.root.user:* hdfs root user
- * *hdfs.root.dir:* HDFS root, default /slidertst
- * *hdfs.user.dir:* HDFS user dir, default /user
- * *test.root:* local test root folder, default /test
- * *cluster.name:* name of the test cluster, default tst1
- * *cluster.type:* cluster type to build and test, e.g. hbase,storm,accumulo
-
-* **agent**
diff --git a/slider-funtest/src/test/resources/log4j.properties b/slider-funtest/src/test/resources/log4j.properties
new file mode 100644
index 0000000..a552a55
--- /dev/null
+++ b/slider-funtest/src/test/resources/log4j.properties
@@ -0,0 +1,59 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# log4j configuration used during build and unit tests
+
+log4j.rootLogger=INFO,stdout
+log4j.threshhold=ALL
+log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} (%F:%M(%L)) - %m%n
+
+log4j.appender.subprocess=org.apache.log4j.ConsoleAppender
+log4j.appender.subprocess.layout=org.apache.log4j.PatternLayout
+log4j.appender.subprocess.layout.ConversionPattern=[%c{1}]: %m%n
+#log4j.logger.org.apache.slider.yarn.appmaster.SliderAppMasterer.master=INFO,subprocess
+
+log4j.logger.org.apache.slider=DEBUG
+log4j.logger.org.apache.hadoop.yarn.service.launcher=DEBUG
+
+
+
+#log4j.logger.org.apache.hadoop.yarn.service.launcher=DEBUG
+#log4j.logger.org.apache.hadoop.yarn.service=DEBUG
+#log4j.logger.org.apache.hadoop.yarn.client=DEBUG
+
+#crank back on some noise
+log4j.logger.org.apache.hadoop.util.NativeCodeLoader=ERROR
+log4j.logger.org.apache.hadoop.hdfs.server.datanode.BlockPoolSliceScanner=WARN
+log4j.logger.org.apache.hadoop.hdfs.server.blockmanagement=WARN
+log4j.logger.org.apache.hadoop.hdfs.server.namenode.FSNamesystem.audit=WARN
+log4j.logger.org.apache.hadoop.hdfs=WARN
+
+
+log4j.logger.org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor=WARN
+log4j.logger.org.apache.hadoop.yarn.server.nodemanager.NodeStatusUpdaterImpl=WARN
+log4j.logger.org.apache.zookeeper=WARN
+log4j.logger.org.apache.zookeeper.ClientCnxn=FATAL
+
+log4j.logger.org.apache.hadoop.yarn.server.resourcemanager.security=WARN
+log4j.logger.org.apache.hadoop.metrics2=ERROR
+log4j.logger.org.apache.hadoop.util.HostsFileReader=WARN
+log4j.logger.org.apache.hadoop.yarn.event.AsyncDispatcher=WARN
+log4j.logger.org.apache.hadoop.security.token.delegation=WARN
+log4j.logger.org.apache.hadoop.yarn.util.AbstractLivelinessMonitor=WARN
+log4j.logger.org.apache.hadoop.yarn.server.nodemanager.security=WARN
+log4j.logger.org.apache.hadoop.yarn.server.resourcemanager.RMNMInfo=WARN
diff --git a/src/site/markdown/developing/agent_test_setup.md b/src/site/markdown/developing/agent_test_setup.md
index b0270a0..2ecd2de 100644
--- a/src/site/markdown/developing/agent_test_setup.md
+++ b/src/site/markdown/developing/agent_test_setup.md
@@ -40,8 +40,8 @@
## Permissions
Ensure that the user creating the hbase cluster has necessary permission for the resource management library and the application spec. Perform necessary **chown** and **chmod**.
-1. /share/hbase/hbase-0.96.1-hadoop2/conf
-2. /usr/lib/python2.6/site-packages/resource_management
-3. /var/lib/ambari-agent/cache/stacks/HDP
-4. /var/log/hbase, /var/run/hbase (or appropriate log and run directories)
+1. `/share/hbase/hbase-0.96.1-hadoop2/conf`
+2. `/usr/lib/python2.6/site-packages/resource_management`
+3. `/var/lib/ambari-agent/cache/stacks/HDP`
+4. `/var/log/hbase` , `/var/run/hbase` (or appropriate log and run directories)
5. Ensure hbase root/staging HDFS directories have appropriate permission
diff --git a/src/site/markdown/developing/building.md b/src/site/markdown/developing/building.md
index ae5a950..8bf3954 100644
--- a/src/site/markdown/developing/building.md
+++ b/src/site/markdown/developing/building.md
@@ -27,6 +27,10 @@
The network on the development system must be functional, with hostname lookup
of the local host working. Tests will fail without this.
+### Java 7
+
+Slider is built on Java 7 -please have a JDK for Java 7 or 8 set up
+
### Maven
You will need a version of Maven 3.0+, set up with enough memory
@@ -39,12 +43,12 @@
### Protoc
-You need a copy of the `protoc` compile
+You need a copy of the `protoc` compiler for protobuf compilation
1. OS/X: `brew install protobuf`
1. Others: consult (Building Hadoop documentation)[http://wiki.apache.org/hadoop/HowToContribute].
-The version of protoc installed must be the same as that used by Hadoop itself.
+The version of `protoc` installed must be the same as that used by Hadoop itself.
This is absolutely critical to prevent JAR version problems.
## Building a compatible Hadoop version
@@ -297,9 +301,10 @@
# Development Notes
-
+<!---
## Git branch model
+
The git branch model uses is
[Git Flow](http://nvie.com/posts/a-successful-git-branching-model/).
@@ -328,8 +333,9 @@
git flow release start 0.4.0
git flow release finish 0.4.0
-
-## Attn OS/X developers
+-->
+
+## Attention OS/X developers
YARN on OS/X doesn't terminate subprocesses the way it does on Linux, so
HBase Region Servers created by the hbase shell script remain running
diff --git a/src/site/markdown/developing/functional_tests.md b/src/site/markdown/developing/functional_tests.md
new file mode 100644
index 0000000..cf8ef11
--- /dev/null
+++ b/src/site/markdown/developing/functional_tests.md
@@ -0,0 +1,373 @@
+<!---
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+# Testing
+
+ The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL
+ NOT", "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and
+ "OPTIONAL" in this document are to be interpreted as described in
+ RFC 2119.
+
+# Functional Tests
+
+The functional test suite is designed to run the executables against
+a live cluster.
+
+For these to work you need
+1. A YARN Cluster -secure or insecure
+1. A `slider-client.xml` file configured to interact with the cluster
+1. Agent
+1. HBase tests: HBase `.tar.gz` uploaded to HDFS, and a local or remote accumulo conf
+directory
+1. Accumulo Tests Accumulo `.tar.gz` uploaded to HDFS, and a local or remote accumulo conf
+directory
+
+## Configuration of functional tests
+
+Maven needs to be given
+1. A path to the expanded test archive
+1. A path to a slider configuration directory for the cluster
+
+The path for the expanded test is automatically calculated as being the directory under
+`..\slider-assembly\target` where an untarred slider distribution can be found.
+If it is not present, the tests will fail
+
+The path to the configuration directory must be supplied in the property
+`slider.conf.dir` which can be set on the command line
+
+ mvn test -Dslider.conf.dir=src/test/configs/sandbox/slider
+
+It can also be set in the (optional) file `slider-funtest/build.properties`:
+
+ slider.conf.dir=src/test/configs/sandbox/slider
+
+This file is loaded whenever a slider build or test run takes place
+
+## Configuration of `slider-client.xml`
+
+The `slider-client.xml` must have extra configuration options for both the HBase and
+Accumulo tests, as well as a common set for actually talking to a YARN cluster.
+
+## Disabling the functional tests entirely
+
+All functional tests which require a live YARN cluster
+can be disabled through the property `slider.funtest.enabled`
+
+ <property>
+ <name>slider.funtest.enabled</name>
+ <value>false</value>
+ </property>
+
+There is a configuration do do exactly this in
+`src/test/configs/offline/slider`:
+
+ slider.conf.dir=src/test/configs/offline/slider
+
+Tests which do not require a live YARN cluster will still run;
+these verify that the `bin/slider` script works.
+
+### Non-mandatory options
+
+The following test options may be added to `slider-client.xml` if the defaults
+need to be changed
+
+ <property>
+ <name>slider.test.zkhosts</name>
+ <description>comma separated list of ZK hosts</description>
+ <value>localhost</value>
+ </property>
+
+ <property>
+ <name>slider.test.thaw.wait.seconds</name>
+ <description>Time to wait in seconds for a thaw to result in a running AM</description>
+ <value>60000</value>
+ </property>
+
+ <property>
+ <name>slider.test.freeze.wait.seconds</name>
+ <description>Time to wait in seconds for a freeze to halt the cluster</description>
+ <value>60000</value>
+ </property>
+
+ <property>
+ <name>slider.test.timeout.millisec</name>
+ <description>Time out in milliseconds before a test is considered to have failed.
+ There are some maven properties which also define limits and may need adjusting</description>
+ <value>180000</value>
+ </property>
+
+
+
+Note that while the same properties need to be set in
+`slider-core/src/test/resources/slider-client.xml`, those tests take a file in the local
+filesystem -here a URI to a path visible across all nodes in the cluster are required
+the tests do not copy the .tar/.tar.gz files over. The application configuration
+directories may be local or remote -they are copied into the `.slider` directory
+during cluster creation.
+
+## Provider-specific parameters
+
+An individual provider can pick up settings from their own
+`src/test/resources/slider-client.xml` file, or the one in `slider-core`.
+We strongly advice placing all the values in the `slider-core` file.
+
+1. All uncertainty about which file is picked up on the class path first goes
+away
+2. There's one place to keep all the configuration values in sync.
+
+### Agent Tests
+
+ <property>
+ <name>slider.test.agent.enabled</name>
+ <description>Flag to enable/disable Agent tests</description>
+ <value>true</value>
+ </property>
+
+The agent tarball needs to be pushed to HBase. This is the tar
+file created
+
+ <property>
+ <name>slider.test.agent.tar</name>
+ <description>Path to the Agent Tar file in HDFS</description>
+ <value>hdfs://sandbox:8020/user/slider/agent.tar.gz</value>
+ </property>
+
+
+### HBase Tests
+
+The HBase tests can be enabled or disabled
+
+ <property>
+ <name>slider.test.hbase.enabled</name>
+ <description>Flag to enable/disable HBase tests</description>
+ <value>true</value>
+ </property>
+
+Mandatory test parameters must be added to `slider-client.xml`
+
+
+ <property>
+ <name>slider.test.hbase.tar</name>
+ <description>Path to the HBase Tar file in HDFS</description>
+ <value>hdfs://sandbox:8020/user/slider/hbase.tar.gz</value>
+ </property>
+
+ <property>
+ <name>slider.test.hbase.appconf</name>
+ <description>Path to the directory containing the HBase application config</description>
+ <value>file://${user.dir}/src/test/configs/sandbox/hbase</value>
+ </property>
+
+Optional parameters:
+
+ <property>
+ <name>slider.test.hbase.launch.wait.seconds</name>
+ <description>Time to wait in seconds for HBase to start</description>
+ <value>180000</value>
+ </property>
+
+
+#### Accumulo configuration options
+
+Enable/disable the tests
+
+ <property>
+ <name>slider.test.accumulo.enabled</name>
+ <description>Flag to enable/disable Accumulo tests</description>
+ <value>true</value>
+ </property>
+
+
+Optional parameters
+
+ <property>
+ <name>slider.test.accumulo.launch.wait.seconds</name>
+ <description>Time to wait in seconds for Accumulo to start</description>
+ <value>180000</value>
+ </property>
+
+
+
+### Configuring the YARN cluster for tests
+
+
+Here are the configuration options we use in `yarn-site.xml` for testing:
+
+These tell YARN to ignore memory requirements in allocating VMs, and
+to keep the log files around after an application run.
+
+ <property>
+ <name>yarn.scheduler.minimum-allocation-mb</name>
+ <value>1</value>
+ </property>
+ <property>
+ <description>Whether physical memory limits will be enforced for
+ containers.
+ </description>
+ <name>yarn.nodemanager.pmem-check-enabled</name>
+ <value>false</value>
+ </property>
+ <!-- we really don't want checking here-->
+ <property>
+ <name>yarn.nodemanager.vmem-check-enabled</name>
+ <value>false</value>
+ </property>
+
+ <!-- how long after a failure to see what is left in the directory-->
+ <property>
+ <name>yarn.nodemanager.delete.debug-delay-sec</name>
+ <value>60000</value>
+ </property>
+
+ <!--ten seconds before the process gets a -9 -->
+ <property>
+ <name>yarn.nodemanager.sleep-delay-before-sigkill.ms</name>
+ <value>30000</value>
+ </property>
+
+
+### Testing against a secure cluster
+
+To test against a secure cluster
+
+1. `slider-client.xml` must be configured as per [Security](../security.html).
+1. the client must have the kerberos tokens issued so that the user running
+the tests has access to HDFS and YARN.
+
+If there are problems authenticating (including the cluster being offline)
+the tests appear to hang
+
+### Validating the configuration
+
+ mvn test -Dtest=TestBuildSetup
+
+### Using relative paths in test configurations
+
+When you are sharing configurations across machines via SCM or similar,
+its impossible to have absolute paths in the configuration options to
+the location of items in the local filesystem (e.g. configuration directories).
+
+There's two techniques
+
+1. Keep the data in HDFS and refer to it there. This works if there is a shared,
+persistent HDFS cluster.
+
+1. Use the special property `slider.test.conf.dir` that is set to the path
+of the directory, and which can then be used to create an absolute path
+from paths relative to the configuration dir:
+
+ <property>
+ <name>slider.test.hbase.appconf</name>
+ <description>Path to the directory containing the HBase application config</description>
+ <value>file://${slider.test.conf.dir}/../hbase</value>
+ </property>
+
+
+If the actual XML file path is required, a similar property
+`slider.test.conf.xml` is set.
+
+
+## Parallel execution
+
+Attempts to run test cases in parallel failed -even with a configuration
+to run methods in a class sequentially, but separate classes independently.
+
+Even after identifying and eliminating some unintended sharing of static
+mutable variables, trying to run test cases in parallel seemed to hang
+tests and produce timeouts.
+
+For this reason parallel tests have been disabled. To accelerate test runs
+through parallelization, run different tests on different hosts instead.
+
+## Other constraints
+
+* Port assignments SHOULD NOT be fixed, as this will cause clusters to fail if
+there are too many instances of a role on a same host, or if other tests are
+using the same port.
+* If a test does need to fix a port, it MUST be for a single instance of a role,
+and it must be different from all others. The assignment should be set in
+`org.apache.slider.funtest.itest.PortAssignments` so as to ensure uniqueness
+over time. Otherwise: use the value of `0` to allow the OS to assign free ports
+on demand.
+
+## Test Requirements
+
+
+1. Test cases should be written so that each class works with exactly one
+Slider-deployed cluster
+1. Every test MUST have its own cluster name -preferably derived from the
+classname.
+1. This cluster should be deployed in an `@BeforeClass` method.
+1. The `@AfterClass` method MUST tear this cluster down.
+1. Tests must skip their execution if functional tests -or the
+specific hbase or accumulo categories- are disabled.
+1. Tests within the suite (i.e. class) must be designed to be independent
+-to work irrespectively of the ordering of other tests.
+
+## Running and debugging the functional tests.
+
+The functional tests all
+
+1. In the root `slider` directory, build a complete Slider release
+
+ mvn install -DskipTests
+1. Start the YARN cluster/set up proxies to connect to it, etc.
+
+1. In the `slider-funtest` dir, run the test
+
+ mvn test -Dtest=TestHBaseCreateCluster
+
+A common mistake during development is to rebuild the `slider-core` JARs
+then the `slider-funtest` tests without rebuilding the `slider-assembly`.
+In this situation, the tests are in sync with the latest build of the code
+-including any bug fixes- but the scripts executed by those tests are
+of a previous build of `slider-core.jar`. As a result, the fixes are not picked
+up.
+
+#### To propagate changes in slider-core through to the funtest classes for
+testing, you must build/install all the slider packages from the root assembly.
+
+ mvn clean install -DskipTests
+
+## Limitations of slider-funtest
+
+1. All tests run from a single client -workload can't scale
+1. Output from failed AM and containers aren't collected
+
+## Troubleshooting the functional tests
+
+1. If you are testing in a local VM and stops responding, it'll have been
+swapped out to RAM. Rebooting can help, but for a long term fix go through
+all the Hadoop configurations (HDFS, YARN, Zookeeper) and set their heaps to
+smaller numbers, like 256M each. Also: turn off unused services (hcat, oozie,
+webHDFS)
+
+1. The YARN UI will list the cluster launches -look for the one
+with a name close to the test and view its logs
+
+1. Container logs will appear "elsewhere". The log lists
+the containers used -you may be able to track the logs
+down from the specific nodes.
+
+1. If you browse the filesystem, look for the specific test clusters
+in `~/.slider/cluster/$testname`
+
+1. If you are using a secure cluster, make sure that the clocks
+are synchronized, and that you have a current token -`klist` will
+tell you this. In a VM: install and enable `ntp`, consider rebooting if ther
+are any problems. Check also that it has the same time zone settings
+as the host OS.
diff --git a/src/site/markdown/developing/index.md b/src/site/markdown/developing/index.md
index 34ca20a..0bd0659 100644
--- a/src/site/markdown/developing/index.md
+++ b/src/site/markdown/developing/index.md
@@ -23,6 +23,8 @@
* [Building](building.html)
* [Debugging](debugging.html)
* [Testing](testing.html)
+* [Functional Testing](functional_tests.html)
+* [Manual Testing](manual_testing.html)
* [Agent test setup](agent_test_setup.html)
* [Releasing](releasing.html)
diff --git a/src/site/markdown/developing/manual_testing.md b/src/site/markdown/developing/manual_testing.md
new file mode 100644
index 0000000..aa32446
--- /dev/null
+++ b/src/site/markdown/developing/manual_testing.md
@@ -0,0 +1,53 @@
+<!---
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+# Manual Testing
+
+Manual testing invloves using Slider package and an AppPackage to perform basic
+ cluster functionalities such as create/destroy, flex up/down, and freeze/thaw.
+ A python helper script is provided that can be used to automatically test and app package.
+
+## `SliderTester.py`
+Details to be added.
+
+## `SliderTester.ini`
+The various config parameters are:
+
+### slider
+* `package`: location of the slider package
+* `jdk.path`: jdk path on the test hosts
+
+### app
+* `package`: location of the app package
+
+### cluster
+* `yarn.application.classpath`: yarn application classpaths
+* `slider.zookeeper.quorum`: the ZK quorum hosts
+* `yarn.resourcemanager.address`:
+* `yarn.resourcemanager.scheduler.address`:
+* `fs.defaultFS`: e.g. `hdfs://NN_HOST:8020`
+
+### test
+* `app.user`: user to use for app creation
+* `hdfs.root.user`: hdfs root user
+* `hdfs.root.dir`: HDFS root, default /slidertst
+* `hdfs.user.dir`: HDFS user dir, default /user
+* `test.root`: local test root folder, default /test
+* `cluster.name`: name of the test cluster, default tst1
+* `cluster.type`: cluster type to build and test, e.g. hbase,storm,accumulo
+
+### agent
diff --git a/src/site/markdown/developing/testing.md b/src/site/markdown/developing/testing.md
index 9592a76..2ad2bdf 100644
--- a/src/site/markdown/developing/testing.md
+++ b/src/site/markdown/developing/testing.md
@@ -27,19 +27,25 @@
Slider core contains a suite of tests that are designed to run on the local machine,
using Hadoop's `MiniDFSCluster` and `MiniYARNCluster` classes to create small,
one-node test clusters. All the YARN/HDFS code runs in the JUnit process; the
-AM and spawned HBase and Accumulo processes run independently.
+AM and spawned processeses run independently.
+
+
+
+### For HBase Tests in `slider-providers/hbase`
Requirements
-* A copy of hbase.tar.gz in the local filesystem
-* A an expanded hbase.tar.gz in the local filesystem
+* A copy of `hbase.tar.gz` in the local filesystem
+* A an expanded `hbase.tar.gz` in the local filesystem
-* A copy of accumulo.tar.gz in the local filesystem,
-* An expanded accumulo.tar.gz in the local filesystem,
+
+### For Accumulo Tests in `slider-providers/accumulo`
+* A copy of `accumulo.tar.gz` in the local filesystem,
+* An expanded `accumulo.tar.gz` in the local filesystem,
* an expanded Zookeeper installation
All of these need to be defined in the file `slider-core/src/test/resources/slider-test.xml`
-Here's
+Example:
<configuration>
@@ -108,106 +114,7 @@
nodes in the cluster -which usually means HDFS, and so an `hdfs://` URL
-
-## Functional Tests
-
-The functional test suite is designed to run the executables against
-a live cluster.
-
-For these to work you need
-1. A YARN Cluster -secure or insecure
-1. A `slider-client.xml` file configured to interact with the cluster
-1. HBase `.tar.gz` uploaded to HDFS, and a local or remote accumulo conf
-directory
-1. Accumulo `.tar.gz` uploaded to HDFS, and a local or remote accumulo conf
-directory
-
-## Configuration of functional tests
-
-Maven needs to be given
-1. A path to the expanded test archive
-1. A path to a slider configuration directory for the cluster
-
-The path for the expanded test is automatically calculated as being the directory under
-`..\slider-assembly\target` where an untarred slider distribution can be found.
-If it is not present, the tests will fail
-
-The path to the configuration directory must be supplied in the property
-`slider.conf.dir` which can be set on the command line
-
- mvn test -Dslider.conf.dir=src/test/configs/sandbox/slider
-
-It can also be set in the (optional) file `slider-funtest/build.properties`:
-
- slider.conf.dir=src/test/configs/sandbox/slider
-
-This file is loaded whenever a slider build or test run takes place
-
-## Configuration of `slider-client.xml`
-
-The `slider-client.xml` must have extra configuration options for both the HBase and
-Accumulo tests, as well as a common set for actually talking to a YARN cluster.
-
-## Disabling the functional tests entirely
-
-All functional tests which require a live YARN cluster
-can be disabled through the property `slider.funtest.enabled`
-
- <property>
- <name>slider.funtest.enabled</name>
- <value>false</value>
- </property>
-
-There is a configuration do do exactly this in
-`src/test/configs/offline/slider`:
-
- slider.conf.dir=src/test/configs/offline/slider
-
-Tests which do not require a live YARN cluster will still run;
-these verify that the `bin/slider` script works.
-
-### Non-mandatory options
-
-The following test options may be added to `slider-client.xml` if the defaults
-need to be changed
-
- <property>
- <name>slider.test.zkhosts</name>
- <description>comma separated list of ZK hosts</description>
- <value>localhost</value>
- </property>
-
- <property>
- <name>slider.test.thaw.wait.seconds</name>
- <description>Time to wait in seconds for a thaw to result in a running AM</description>
- <value>60000</value>
- </property>
-
- <property>
- <name>slider.test.freeze.wait.seconds</name>
- <description>Time to wait in seconds for a freeze to halt the cluster</description>
- <value>60000</value>
- </property>
-
- <property>
- <name>slider.test.timeout.millisec</name>
- <description>Time out in milliseconds before a test is considered to have failed.
- There are some maven properties which also define limits and may need adjusting</description>
- <value>180000</value>
- </property>
-
-
-
-Note that while the same properties need to be set in
-`slider-core/src/test/resources/slider-client.xml`, those tests take a file in the local
-filesystem -here a URI to a path visible across all nodes in the cluster are required
-the tests do not copy the .tar/.tar.gz files over. The application configuration
-directories may be local or remote -they are copied into the `.slider` directory
-during cluster creation.
-
-##
-
-Provider-specific parameters
+## Provider-specific parameters
An individual provider can pick up settings from their own
`src/test/resources/slider-client.xml` file, or the one in `slider-core`.
@@ -217,7 +124,10 @@
away
2. There's one place to keep all the configuration values in sync.
-### HBase Parameters
+### Agent Tests
+
+
+### HBase Tests
The HBase tests can be enabled or disabled
@@ -270,172 +180,3 @@
<value>180000</value>
</property>
-
-
-### Configuring the YARN cluster for tests
-
-
-Here are the configuration options we use in `yarn-site.xml` for testing:
-
-These tell YARN to ignore memory requirements in allocating VMs, and
-to keep the log files around after an application run.
-
- <property>
- <name>yarn.scheduler.minimum-allocation-mb</name>
- <value>1</value>
- </property>
- <property>
- <description>Whether physical memory limits will be enforced for
- containers.
- </description>
- <name>yarn.nodemanager.pmem-check-enabled</name>
- <value>false</value>
- </property>
- <!-- we really don't want checking here-->
- <property>
- <name>yarn.nodemanager.vmem-check-enabled</name>
- <value>false</value>
- </property>
-
- <!-- how long after a failure to see what is left in the directory-->
- <property>
- <name>yarn.nodemanager.delete.debug-delay-sec</name>
- <value>60000</value>
- </property>
-
- <!--ten seconds before the process gets a -9 -->
- <property>
- <name>yarn.nodemanager.sleep-delay-before-sigkill.ms</name>
- <value>30000</value>
- </property>
-
-
-### Testing against a secure cluster
-
-To test against a secure cluster
-
-1. `slider-client.xml` must be configured as per [Security](security.html).
-1. the client must have the kerberos tokens issued so that the user running
-the tests has access to HDFS and YARN.
-
-If there are problems authenticating (including the cluster being offline)
-the tests appear to hang
-
-### Validating the configuration
-
- mvn test -Dtest=TestBuildSetup
-
-### Using relative paths in test configurations
-
-When you are sharing configurations across machines via SCM or similar,
-its impossible to have absolute paths in the configuration options to
-the location of items in the local filesystem (e.g. configuration directories).
-
-There's two techniques
-
-1. Keep the data in HDFS and refer to it there. This works if there is a shared,
-persistent HDFS cluster.
-
-1. Use the special property `slider.test.conf.dir` that is set to the path
-of the directory, and which can then be used to create an absolute path
-from paths relative to the configuration dir:
-
- <property>
- <name>slider.test.hbase.appconf</name>
- <description>Path to the directory containing the HBase application config</description>
- <value>file://${slider.test.conf.dir}/../hbase</value>
- </property>
-
-
-If the actual XML file path is required, a similar property
-`slider.test.conf.xml` is set.
-
-
-## Parallel execution
-
-Attempts to run test cases in parallel failed -even with a configuration
-to run methods in a class sequentially, but separate classes independently.
-
-Even after identifying and eliminating some unintended sharing of static
-mutable variables, trying to run test cases in parallel seemed to hang
-tests and produce timeouts.
-
-For this reason parallel tests have been disabled. To accelerate test runs
-through parallelization, run different tests on different hosts instead.
-
-## Other constraints
-
-* Port assignments SHOULD NOT be fixed, as this will cause clusters to fail if
-there are too many instances of a role on a same host, or if other tests are
-using the same port.
-* If a test does need to fix a port, it MUST be for a single instance of a role,
-and it must be different from all others. The assignment should be set in
-`org.apache.slider.funtest.itest.PortAssignments` so as to ensure uniqueness
-over time. Otherwise: use the value of `0` to allow the OS to assign free ports
-on demand.
-
-## Test Requirements
-
-
-1. Test cases should be written so that each class works with exactly one
-Slider-deployed cluster
-1. Every test MUST have its own cluster name -preferably derived from the
-classname.
-1. This cluster should be deployed in an `@BeforeClass` method.
-1. The `@AfterClass` method MUST tear this cluster down.
-1. Tests must skip their execution if functional tests -or the
-specific hbase or accumulo categories- are disabled.
-1. Tests within the suite (i.e. class) must be designed to be independent
--to work irrespectively of the ordering of other tests.
-
-## Running and debugging the functional tests.
-
-The functional tests all
-
-1. In the root `slider` directory, build a complete Slider release
-
- mvn install -DskipTests
-1. Start the YARN cluster/set up proxies to connect to it, etc.
-
-1. In the `slider-funtest` dir, run the test
-
- mvn test -Dtest=TestHBaseCreateCluster
-
-A common mistake during development is to rebuild the `slider-core` JARs
-then the `slider-funtest` tests without rebuilding the `slider-assembly`.
-In this situation, the tests are in sync with the latest build of the code
--including any bug fixes- but the scripts executed by those tests are
-of a previous build of `slider-core.jar`. As a result, the fixes are not picked
-up.
-
-#### To propagate changes in slider-core through to the funtest classes for
-testing, you must build/install all the slider packages from the root assembly.
-
- mvn clean install -DskipTests
-
-## Limitations of slider-funtest
-
-1. All tests run from a single client -workload can't scale
-1. Output from failed AM and containers aren't collected
-
-## Troubleshooting the functional tests
-
-1. If you are testing in a local VM and stops responding, it'll have been
-swapped out to RAM. Rebooting can help, but for a long term fix go through
-all the Hadoop configurations (HDFS, YARN, Zookeeper) and set their heaps to
-smaller numbers, like 256M each. Also: turn off unused services (hcat, oozie,
-webHDFS)
-
-1. The YARN UI will list the cluster launches -look for the one
-with a name close to the test and view its logs
-
-1. Container logs will appear "elsewhere". The log lists
-the containers used -you may be able to track the logs
-down from the specific nodes.
-
-1. If you browse the filesystem, look for the specific test clusters
-in `~/.slider/cluster/$testname`
-
-1. If you are using a secure cluster, make sure that the clocks
-are synchronized, and that you have a current token -`klist` will
-tell you this. In a VM: install and enable `ntp`.
diff --git a/src/test/clusters/sandbox/slider/slider-client.xml b/src/test/clusters/sandbox/slider/slider-client.xml
index 2811da8..2dd3f51 100644
--- a/src/test/clusters/sandbox/slider/slider-client.xml
+++ b/src/test/clusters/sandbox/slider/slider-client.xml
@@ -50,6 +50,19 @@
/etc/hadoop/conf,/usr/lib/hadoop/*,/usr/lib/hadoop/lib/*,/usr/lib/hadoop-hdfs/*,/usr/lib/hadoop-hdfs/lib/*,/usr/lib/hadoop-yarn/*,/usr/lib/hadoop-yarn/lib/*,/usr/lib/hadoop-mapreduce/*,/usr/lib/hadoop-mapreduce/lib/*
</value>
</property>
+
+ <property>
+ <name>slider.test.agent.enabled</name>
+ <description>Flag to enable/disable Agent tests</description>
+ <value>true</value>
+ </property>
+
+
+ <property>
+ <name>slider.test.hbase.enabled</name>
+ <description>Flag to enable/disable HBase tests</description>
+ <value>true</value>
+ </property>
<property>
<name>slider.test.hbase.tar</name>
@@ -91,7 +104,7 @@
<description>Path to the directory containing the Accumulo application
config
</description>
- <value>file:///home/slider/projects//Projects/slider/test-configs/sandbox/accumulo</value>
+ <value>file:///home/slider/projects/Projects/slider/test-configs/sandbox/accumulo</value>
</property>
