SLIDER 30. Clean the agent functional test and consolidate steps
git-svn-id: https://svn.apache.org/repos/asf/incubator/slider/trunk@1593187 13f79535-47bb-0310-9956-ffa450edef68
diff --git a/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py b/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py
index a72cf87..f114d03 100644
--- a/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py
+++ b/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py
@@ -145,10 +145,24 @@
override_output_files = False
if command['roleCommand'] == "GET_CONFIG":
- logger.info("Requesting applied config ...")
- return {
- 'configurations': self.applied_configs
- }
+ if 'commandParams' in command and 'config_type' in command['commandParams']:
+ config_type = command['commandParams']['config_type']
+ logger.info("Requesting applied config for type {0}".format(config_type))
+ if config_type in self.applied_configs:
+ return {
+ 'configurations': {config_type: self.applied_configs[config_type]}
+ }
+ else:
+ return {
+ 'configurations': {}
+ }
+ pass
+ else:
+ logger.info("Requesting all applied config.")
+ return {
+ 'configurations': self.applied_configs
+ }
+ pass
else:
res = self.runCommand(command, self.status_commands_stdout,
diff --git a/slider-agent/src/test/python/agent/TestCustomServiceOrchestrator.py b/slider-agent/src/test/python/agent/TestCustomServiceOrchestrator.py
index 9f04298..2abbd58 100644
--- a/slider-agent/src/test/python/agent/TestCustomServiceOrchestrator.py
+++ b/slider-agent/src/test/python/agent/TestCustomServiceOrchestrator.py
@@ -34,7 +34,6 @@
from mock.mock import MagicMock, patch
import StringIO
import sys
-from AgentException import AgentException
class TestCustomServiceOrchestrator(TestCase):
@@ -209,6 +208,14 @@
'commandType': 'STATUS_COMMAND'
}
+ command_get_specific = {
+ 'roleCommand': 'GET_CONFIG',
+ 'commandType': 'STATUS_COMMAND',
+ 'commandParams': {
+ 'config_type': 'hbase-site'
+ }
+ }
+
tempdir = tempfile.gettempdir()
config = MagicMock()
config.get.return_value = "something"
@@ -231,6 +238,11 @@
'hbase.log': tempdir, 'hbase.number': '10485760'},
'hbase-log4j': {'a': 'b'}}
+ expected_specific = {
+ 'hbase-site': {
+ 'hbase.log': tempdir, 'hbase.number': '10485760'},
+ }
+
ret = orchestrator.runCommand(command, "out.txt", "err.txt", True, True)
self.assertEqual(ret['exitcode'], 0)
self.assertTrue(run_file_mock.called)
@@ -238,6 +250,9 @@
ret = orchestrator.requestComponentStatus(command_get)
self.assertEqual(ret['configurations'], expected)
+
+ ret = orchestrator.requestComponentStatus(command_get_specific)
+ self.assertEqual(ret['configurations'], expected_specific)
pass
@patch.object(CustomServiceOrchestrator, "runCommand")
@@ -248,7 +263,7 @@
"clusterName": "",
"componentName": "DATANODE",
'configurations': {},
- 'roleCommand' : "STATUS"
+ 'roleCommand': "STATUS"
}
dummy_controller = MagicMock()
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentCommandTestBase.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentCommandTestBase.groovy
index 36fab27..1f22d3a 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentCommandTestBase.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentCommandTestBase.groovy
@@ -38,25 +38,46 @@
protected static String APP_RESOURCE = "../slider-core/src/test/app_packages/test_command_log/resources.json"
protected static String APP_TEMPLATE = "../slider-core/src/test/app_packages/test_command_log/appConfig.json"
- protected static final File LOCAL_SLIDER_AGENT_TARGZ
-
+ protected static String APP_PKG = "../slider-core/src/test/app_packages/test_command_log/cmd_log_app_pkg.tar"
+ protected static String AGENT_CONF = "../slider-agent/conf/agent.ini"
+ protected static final File LOCAL_SLIDER_AGENT_TARGZ
+ protected static final File LOCAL_APP_PKZ
+ protected static final File LOCAL_AGENT_CONF
+
protected static Path agentTarballPath;
-
+ protected static Path appPkgPath;
+ protected static Path agtIniPath;
+
static {
AGENTTESTS_ENABLED = SLIDER_CONFIG.getBoolean(KEY_TEST_AGENT_ENABLED, false)
LOCAL_SLIDER_AGENT_TARGZ = new File(
SLIDER_BIN_DIRECTORY,
AGENT_SLIDER_GZ).canonicalFile
+ LOCAL_APP_PKZ = new File(APP_PKG).canonicalFile
+ LOCAL_AGENT_CONF = new File(AGENT_CONF).canonicalFile
}
@BeforeClass
public static void setupAgent() {
assumeAgentTestsEnabled()
- assume(LOCAL_SLIDER_AGENT_TARGZ.exists(), "Slider agent not found at $LOCAL_SLIDER_AGENT_TARGZ")
- agentTarballPath = new Path(clusterFS.homeDirectory, "agent.tar.gz")
+ // Upload the agent tarball
+ assume(LOCAL_SLIDER_AGENT_TARGZ.exists(), "Slider agent not found at $LOCAL_SLIDER_AGENT_TARGZ")
+ agentTarballPath = new Path(clusterFS.homeDirectory, "/slider/agent/slider-agent.tar.gz")
Path localTarball = new Path(LOCAL_SLIDER_AGENT_TARGZ.toURI());
clusterFS.copyFromLocalFile(false, true, localTarball, agentTarballPath)
+
+ // Upload the app pkg
+ assume(LOCAL_APP_PKZ.exists(), "App pkg not found at $LOCAL_APP_PKZ")
+ appPkgPath = new Path(clusterFS.homeDirectory, "/slider/cmd_log_app_pkg.tar")
+ Path localAppPkg = new Path(LOCAL_SLIDER_AGENT_TARGZ.toURI());
+ clusterFS.copyFromLocalFile(false, true, localAppPkg, appPkgPath)
+
+ // Upload the agent.ini
+ assume(LOCAL_AGENT_CONF.exists(), "Agent config not found at $LOCAL_AGENT_CONF")
+ agtIniPath = new Path(clusterFS.homeDirectory, "/slider/agent/conf/agent.ini")
+ Path localAgtIni = new Path(LOCAL_AGENT_CONF.toURI());
+ clusterFS.copyFromLocalFile(false, true, localAgtIni, agtIniPath)
}
public static void assumeAgentTestsEnabled() {
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/TestAppsThroughAgent.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/TestAppsThroughAgent.groovy
index 836d770..c56056c 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/TestAppsThroughAgent.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/TestAppsThroughAgent.groovy
@@ -42,7 +42,7 @@
}
@Test
- public void testCreateFlexHBase() throws Throwable {
+ public void testCreateFlex() throws Throwable {
if (!AGENTTESTS_ENABLED) {
log.info "TESTS are not run."
return
diff --git a/src/site/markdown/developing/agent_test_setup.md b/src/site/markdown/developing/agent_test_setup.md
deleted file mode 100644
index 2ecd2de..0000000
--- a/src/site/markdown/developing/agent_test_setup.md
+++ /dev/null
@@ -1,47 +0,0 @@
-<!---
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-# Setting up for Agent test
-
- The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL
- NOT", "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and
- "OPTIONAL" in this document are to be interpreted as described in
- RFC 2119.
-
-An agent is an application independent entity that can handle any application as long as its definition conforms to a specific format (TBD). While Slider will install the agent in its entirity, for now the host, where the agent is deployed, needs to be prepared for the agent. The following steps are necessary to setup the host for agent.
-
-## Install resource management library
-The resource management library is a common python library used by all application specification. The library is either pre-installed on the hosts or it is packaged along with the application package itself.
-
-Get resource_management.tar and install it at /usr/lib/python2.6/site-packages
-
-## Install the application spec
-The application spec is the declarative definition of a application that can be run on YARN. *At this point only HBase application is supported.*
-
-Get HDP-2.0.6.tar and install at /var/lib/ambari-agent/cache/stacks/HDP.
-
-## Install HBase using tarball
-Get the hbase tarball, hbase-0.96.1-hadoop2-bin.tar.tar.tar.gz, and expand it at /share/hbase.
-
-## Permissions
-Ensure that the user creating the hbase cluster has necessary permission for the resource management library and the application spec. Perform necessary **chown** and **chmod**.
-
-1. `/share/hbase/hbase-0.96.1-hadoop2/conf`
-2. `/usr/lib/python2.6/site-packages/resource_management`
-3. `/var/lib/ambari-agent/cache/stacks/HDP`
-4. `/var/log/hbase` , `/var/run/hbase` (or appropriate log and run directories)
-5. Ensure hbase root/staging HDFS directories have appropriate permission
diff --git a/src/site/markdown/developing/functional_tests.md b/src/site/markdown/developing/functional_tests.md
index b4a1d76..d4e931e 100644
--- a/src/site/markdown/developing/functional_tests.md
+++ b/src/site/markdown/developing/functional_tests.md
@@ -132,22 +132,60 @@
### Agent Tests
+Agent tests are executed through the following mvn command executed at slider/slider-funtest:
+
+```
+cd slider-funtest
+mvn test -Dslider.conf.dir=../src/test/clusters/remote/slider -Dtest=TestAppsThroughAgent -DfailIfNoTests=false
+```
+
+**Enable/Execute the tests**
+
+To enable the test ensure that *slider.test.agent.enabled* is set to *true*.
+
<property>
<name>slider.test.agent.enabled</name>
<description>Flag to enable/disable Agent tests</description>
<value>true</value>
</property>
-The agent tarball needs to be pushed to HBase. This is the tar
-file created
+**Test setup**
+
+Edit config file src/test/clusters/remote/slider/slider-client.xml and ensure that the host names are accurate for the test cluster.
+
+**User setup**
+
+Ensure that the user, running the test, is present on the cluster against which you are running the tests. The user must be a member of the hadoop group.
+
+E.g. adduser **testuser** -d /home/**testuser** -G hadoop -m
+
+**HDFS Setup**
+
+Set up hdfs folders for slider and test user
+
+* su hdfs
+* hdfs dfs -mkdir /slider
+* hdfs dfs -chown testuser:hdfs /slider
+* hdfs dfs -mkdir /user/testuser
+* hdfs dfs -chown testuser:hdfs /user/testuser
+
+Load up agent package and config
+
+* su **testuser**
+* hdfs dfs -mkdir /slider/agent
+* hdfs dfs -mkdir /slider/agent/conf
+* hdfs dfs -copyFromLocal SLIDER_INSTALL_LOC/agent/conf/agent.ini /slider/agent/conf
+
+Ensure correct host name is provided for the agent tarball.
<property>
<name>slider.test.agent.tar</name>
<description>Path to the Agent Tar file in HDFS</description>
- <value>hdfs://sandbox:8020/user/slider/agent.tar.gz</value>
+ <value>hdfs://NN_HOSTNAME:8020/slider/agent/slider-agent.tar.gz</value>
</property>
+
### HBase Tests
The HBase tests can be enabled or disabled
diff --git a/src/test/clusters/remote/README.md b/src/test/clusters/remote/README.md
deleted file mode 100644
index 8969444..0000000
--- a/src/test/clusters/remote/README.md
+++ /dev/null
@@ -1,60 +0,0 @@
-<!---
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License. See accompanying LICENSE file.
--->
-
- # README
-
-This config is required for tests defined in **org.apache.slider.funtest.lifecycle**.
-
-**Test setup**
-
-Edit config file src/test/clusters/remote/slider/slider-client.xml and ensure that the host names are accurate for the test cluster.
-
-**User setup**
-
-Ensure that the user, running the test, is present on the cluster against which you are running the tests. The user must be a member of the hadoop group.
-
-E.g. adduser **testuser** -d /home/**testuser** -G hadoop -m
-
-**HDFS Setup**
-
-Set up various test folders and load the agent and app packages.
-
-Set up hdfs folders for slider and test user
-
-* su hdfs
-* hdfs dfs -mkdir /slider
-* hdfs dfs -chown testuser:hdfs /slider
-* hdfs dfs -mkdir /user/testuser
-* hdfs dfs -chown testuser:hdfs /user/testuser
-
-Set up agent package and config
-
-* su **testuser**
-* hdfs dfs -mkdir /slider/agent
-* hdfs dfs -mkdir /slider/agent/conf
-* hdfs dfs -copyFromLocal <share>/slider-agent.tar.gz /slider/agent
-* hdfs dfs -copyFromLocal <share>/agent.ini /slider/agent/conf
-
-Add app packages
-
-* hdfs dfs -copyFromLocal slider-core/src/test/app_packages/test_command_log/cmd_log_app_pkg.tar
-
-**Enable/Execute the tests**
-
-To enable the test ensure that *slider.test.agent.enabled* is set to *true*. The tests can be executed through the following mvn command executed at slider/slider-funtest.
-
-```
-mvn test -Dslider.conf.dir=../src/test/clusters/remote/slider -Dtest=TestAppsThroughAgent -DfailIfNoTests=false
-```
-
\ No newline at end of file
diff --git a/src/test/clusters/remote/slider/slider-client.xml b/src/test/clusters/remote/slider/slider-client.xml
index e912239..dcf3ecb 100644
--- a/src/test/clusters/remote/slider/slider-client.xml
+++ b/src/test/clusters/remote/slider/slider-client.xml
@@ -53,12 +53,12 @@
<property>
<name>slider.test.agent.enabled</name>
- <value>false</value>
+ <value>true</value>
</property>
<property>
<name>slider.test.agent.tar</name>
- <value>hdfs://c6403.ambari.apache.org:8020/slider/agent/slider-agent-0.23.0-SNAPSHOT.tar.gz</value>
+ <value>hdfs://c6403.ambari.apache.org:8020/slider/agent/slider-agent.tar.gz</value>
</property>
</configuration>
